#include "gtest/gtest.h" #include "storm-config.h" #include "test/storm_gtest.h" #include "storm-parsers/parser/FormulaParser.h" #include "storm/logic/Formulas.h" #include "storm/models/sparse/StandardRewardModel.h" #include "storm/modelchecker/prctl/SparseDtmcPrctlModelChecker.h" #include "storm/modelchecker/results/ExplicitQuantitativeCheckResult.h" #include "storm-parsers/parser/AutoParser.h" #include "storm-parsers/parser/PrismParser.h" #include "storm/storage/expressions/ExpressionManager.h" #include "storm/api/builder.h" #include "storm-pars/analysis/AssumptionChecker.h" #include "storm-pars/analysis/Order.h" #include "storm/storage/expressions/BinaryRelationExpression.h" #include "storm-pars/transformer/SparseParametricDtmcSimplifier.h" #include "storm-pars/storage/ParameterRegion.h" #include "storm-pars/api/storm-pars.h" #include "storm/api/storm.h" #include "storm-parsers/api/storm-parsers.h" #include "storm-pars/api/region.h" TEST(AssumptionCheckerTest, Brp_no_bisimulation) { std::string programFile = STORM_TEST_RESOURCES_DIR "/pdtmc/brp16_2.pm"; std::string formulaAsString = "P=? [F s=4 & i=N ]"; std::string constantsAsString = ""; //e.g. pL=0.9,TOACK=0.5 // Program and formula storm::prism::Program program = storm::api::parseProgram(programFile); program = storm::utility::prism::preprocess(program, constantsAsString); std::vector> formulas = storm::api::extractFormulasFromProperties(storm::api::parsePropertiesForPrismProgram(formulaAsString, program)); std::shared_ptr> model = storm::api::buildSparseModel(program, formulas)->as>(); std::shared_ptr> dtmc = model->as>(); auto simplifier = storm::transformer::SparseParametricDtmcSimplifier>(*dtmc); ASSERT_TRUE(simplifier.simplify(*(formulas[0]))); model = simplifier.getSimplifiedModel(); dtmc = model->as>(); ASSERT_EQ(dtmc->getNumberOfStates(), 193ull); ASSERT_EQ(dtmc->getNumberOfTransitions(), 383ull); // Create the region storm::storage::ParameterRegion::Valuation lowerBoundaries; storm::storage::ParameterRegion::Valuation upperBoundaries; auto vars = storm::models::sparse::getProbabilityParameters(*dtmc); auto region = storm::api::parseRegion("0.00001 <= pK <= 0.00001, 0.00001 <= pL <= 0.99999", vars); auto checker = storm::analysis::AssumptionChecker(formulas[0], dtmc, region, 3); // Check on samples auto expressionManager = std::make_shared(storm::expressions::ExpressionManager()); expressionManager->declareRationalVariable("7"); expressionManager->declareRationalVariable("5"); auto assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("7").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("5").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::UNKNOWN, checker.checkOnSamples(assumption)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("5").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("7").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("7").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("5").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Equal)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); storm::storage::BitVector above(8); above.set(0); storm::storage::BitVector below(8); below.set(1); storm::storage::BitVector initialMiddle(8); std::vector statesSorted = storm::utility::graph::getTopologicalSort(model->getTransitionMatrix()); auto dummyOrder = new storm::analysis::Order(&above, &below, &initialMiddle, 8, &statesSorted); // Validate assumption EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, dummyOrder)); // EXPECT_FALSE(checker.validated(assumption)); expressionManager->declareRationalVariable("6"); expressionManager->declareRationalVariable("8"); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("6").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("8").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); above = storm::storage::BitVector(13); above.set(12); below = storm::storage::BitVector(13); below.set(9); initialMiddle = storm::storage::BitVector(13); dummyOrder = new storm::analysis::Order(&above, &below, &initialMiddle, 13, &statesSorted); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, dummyOrder)); // EXPECT_EQ(checker.validated(assumption)); // EXPECT_FALSE(checker.valid(assumption)); } TEST(AssumptionCheckerTest, Simple1) { std::string programFile = STORM_TEST_RESOURCES_DIR "/pdtmc/simple1.pm"; std::string formulaAsString = "P=? [F s=3]"; std::string constantsAsString = ""; //e.g. pL=0.9,TOACK=0.5 // Program and formula storm::prism::Program program = storm::api::parseProgram(programFile); program = storm::utility::prism::preprocess(program, constantsAsString); std::vector> formulas = storm::api::extractFormulasFromProperties(storm::api::parsePropertiesForPrismProgram(formulaAsString, program)); std::shared_ptr> model = storm::api::buildSparseModel(program, formulas)->as>(); std::shared_ptr> dtmc = model->as>(); auto simplifier = storm::transformer::SparseParametricDtmcSimplifier>(*dtmc); ASSERT_TRUE(simplifier.simplify(*(formulas[0]))); model = simplifier.getSimplifiedModel(); dtmc = model->as>(); ASSERT_EQ(dtmc->getNumberOfStates(), 5); ASSERT_EQ(dtmc->getNumberOfTransitions(), 8); // Create the region auto vars = storm::models::sparse::getProbabilityParameters(*dtmc); auto region = storm::api::parseRegion("0.00001 <= p <= 0.99999", vars); auto checker = storm::analysis::AssumptionChecker(formulas[0], dtmc, region, 3); auto expressionManager = std::make_shared(storm::expressions::ExpressionManager()); expressionManager->declareRationalVariable("1"); expressionManager->declareRationalVariable("2"); // Checking on samples auto assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Equal)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); } TEST(AssumptionCheckerTest, Simple2) { std::string programFile = STORM_TEST_RESOURCES_DIR "/pdtmc/simple2.pm"; std::string formulaAsString = "P=? [F s=3]"; std::string constantsAsString = ""; //e.g. pL=0.9,TOACK=0.5 // Program and formula storm::prism::Program program = storm::api::parseProgram(programFile); program = storm::utility::prism::preprocess(program, constantsAsString); std::vector> formulas = storm::api::extractFormulasFromProperties(storm::api::parsePropertiesForPrismProgram(formulaAsString, program)); std::shared_ptr> model = storm::api::buildSparseModel(program, formulas)->as>(); std::shared_ptr> dtmc = model->as>(); auto simplifier = storm::transformer::SparseParametricDtmcSimplifier>(*dtmc); ASSERT_TRUE(simplifier.simplify(*(formulas[0]))); model = simplifier.getSimplifiedModel(); dtmc = model->as>(); ASSERT_EQ(dtmc->getNumberOfStates(), 5); ASSERT_EQ(dtmc->getNumberOfTransitions(), 8); // Create the region auto vars = storm::models::sparse::getProbabilityParameters(*dtmc); auto region = storm::api::parseRegion("0.00001 <= p <= 0.99999", vars); auto checker = storm::analysis::AssumptionChecker(formulas[0], dtmc, region, 3); auto expressionManager = std::make_shared(storm::expressions::ExpressionManager()); expressionManager->declareRationalVariable("1"); expressionManager->declareRationalVariable("2"); storm::storage::BitVector above(5); above.set(3); storm::storage::BitVector below(5); below.set(4); storm::storage::BitVector initialMiddle(5); std::vector statesSorted = storm::utility::graph::getTopologicalSort(model->getTransitionMatrix()); auto order = new storm::analysis::Order(&above, &below, &initialMiddle, 5, &statesSorted); // Checking on samples and validate auto assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::UNKNOWN, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::VALID, checker.validateAssumption(assumption, order)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, order)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Equal)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, order)); } TEST(AssumptionCheckerTest, Simple3) { std::string programFile = STORM_TEST_RESOURCES_DIR "/pdtmc/simple3.pm"; std::string formulaAsString = "P=? [F s=4]"; std::string constantsAsString = ""; //e.g. pL=0.9,TOACK=0.5 // Program and formula storm::prism::Program program = storm::api::parseProgram(programFile); program = storm::utility::prism::preprocess(program, constantsAsString); std::vector> formulas = storm::api::extractFormulasFromProperties(storm::api::parsePropertiesForPrismProgram(formulaAsString, program)); std::shared_ptr> model = storm::api::buildSparseModel(program, formulas)->as>(); std::shared_ptr> dtmc = model->as>(); auto simplifier = storm::transformer::SparseParametricDtmcSimplifier>(*dtmc); ASSERT_TRUE(simplifier.simplify(*(formulas[0]))); model = simplifier.getSimplifiedModel(); dtmc = model->as>(); ASSERT_EQ(6, dtmc->getNumberOfStates()); ASSERT_EQ(12, dtmc->getNumberOfTransitions()); // Create the region auto vars = storm::models::sparse::getProbabilityParameters(*dtmc); auto region = storm::api::parseRegion("0.00001 <= p <= 0.99999", vars); auto checker = storm::analysis::AssumptionChecker(formulas[0], dtmc, region, 3); auto expressionManager = std::make_shared(storm::expressions::ExpressionManager()); expressionManager->declareRationalVariable("1"); expressionManager->declareRationalVariable("2"); storm::storage::BitVector above(6); above.set(4); storm::storage::BitVector below(6); below.set(5); storm::storage::BitVector initialMiddle(6); std::vector statesSorted = storm::utility::graph::getTopologicalSort(model->getTransitionMatrix()); auto order = new storm::analysis::Order(&above, &below, &initialMiddle, 6, &statesSorted); order->add(3); // Checking on samples and validate auto assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::UNKNOWN, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::VALID, checker.validateAssumption(assumption, order)); EXPECT_EQ(storm::analysis::AssumptionStatus::VALID, checker.validateAssumptionSMTSolver(assumption, order)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, order)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumptionSMTSolver(assumption, order)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Equal)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, order)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumptionSMTSolver(assumption, order)); } TEST(AssumptionCheckerTest, Simple4) { std::string programFile = STORM_TEST_RESOURCES_DIR "/pdtmc/simple4.pm"; std::string formulaAsString = "P=? [F s=3]"; std::string constantsAsString = ""; //e.g. pL=0.9,TOACK=0.5 // Program and formula storm::prism::Program program = storm::api::parseProgram(programFile); program = storm::utility::prism::preprocess(program, constantsAsString); std::vector> formulas = storm::api::extractFormulasFromProperties(storm::api::parsePropertiesForPrismProgram(formulaAsString, program)); std::shared_ptr> model = storm::api::buildSparseModel(program, formulas)->as>(); std::shared_ptr> dtmc = model->as>(); auto simplifier = storm::transformer::SparseParametricDtmcSimplifier>(*dtmc); ASSERT_TRUE(simplifier.simplify(*(formulas[0]))); model = simplifier.getSimplifiedModel(); dtmc = model->as>(); ASSERT_EQ(dtmc->getNumberOfStates(), 5); ASSERT_EQ(dtmc->getNumberOfTransitions(), 8); // Create the region auto vars = storm::models::sparse::getProbabilityParameters(*dtmc); auto region = storm::api::parseRegion("0.00001 <= p <= 0.4", vars); auto checker = storm::analysis::AssumptionChecker(formulas[0], dtmc, region, 3); auto expressionManager = std::make_shared(storm::expressions::ExpressionManager()); expressionManager->declareRationalVariable("1"); expressionManager->declareRationalVariable("2"); // Order storm::storage::BitVector above(5); above.set(3); storm::storage::BitVector below(5); below.set(4); storm::storage::BitVector initialMiddle(5); std::vector statesSorted = storm::utility::graph::getTopologicalSort(model->getTransitionMatrix()); auto order = new storm::analysis::Order(&above, &below, &initialMiddle, 5, &statesSorted); auto assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, order)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumptionSMTSolver(assumption, order)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Greater)); EXPECT_EQ(storm::analysis::AssumptionStatus::UNKNOWN, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::VALID, checker.validateAssumption(assumption, order)); EXPECT_EQ(storm::analysis::AssumptionStatus::VALID, checker.validateAssumptionSMTSolver(assumption, order)); assumption = std::make_shared( storm::expressions::BinaryRelationExpression(*expressionManager, expressionManager->getBooleanType(), expressionManager->getVariable("1").getExpression().getBaseExpressionPointer(), expressionManager->getVariable("2").getExpression().getBaseExpressionPointer(), storm::expressions::BinaryRelationExpression::RelationType::Equal)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.checkOnSamples(assumption)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumption(assumption, order)); EXPECT_EQ(storm::analysis::AssumptionStatus::INVALID, checker.validateAssumptionSMTSolver(assumption, order)); }