diff --git a/README.md b/README.md index f9f87e79..71f25a6f 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,10 @@ There are three typical kinds of steps: * __Refinement__ Action refinement allows for hierarchy in scenarios by delegating implementation of a when-step to another scenario. The `:IN:` and `:OUT:` expressions of the when-step contain conditions (checks), but the `:IN:` and `:OUT:` expressions contradict. A single full scenario can be inserted to resolve the contradiction. For a scenario to be a valid refinement, all `:IN:` conditions must match at the current position and the pending `:OUT:` conditions must be satisfied after insertion. The step implementation should check and confirm the end state of the system under test after refinement. +### File structure + +The test suite that has `Treat this test suite model-based` in its setup is considered the _root suite_ for the model. This can be either a [suite directory](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#suite-directories) or a [suite file](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#suite-files). All scenarios (test cases) in the model's root suite and its sub suites are in scope for the model. When a scenario becomes part of the generated trace, it is added directly to the root suite. Therefore, the test report will show a flattened structure. The root suite must have access to all the keywords used in the scenarios, because in Robot Framework the test suite defines the scope for its test cases. + ### Keeping your models clean Clean models start with tidy model info. If multiple expressions are needed, you can separate them in the `*model info*` by using the pipe symbol (`|`) or by starting the next expression on a new line. A single expression cannot be split over multiple lines. Try to keep expressions simple. If expressions start becoming complex, maybe the model data needs an update. @@ -177,7 +181,7 @@ Modifiers can be used on any type of argument: embedded, positional or named. If #### Technicalities -Please note that all modifiers in the scenario are processed before processing the regular `:IN:` and `:OUT:` expressions. This implies that when model data is used in a modifier, that it will use the model data as it is at the start of the scenario. Any updates to the model data during the scenario steps do not affect the possible choices for the example values. +All modifiers in the scenario are processed before processing the regular `:IN:` and `:OUT:` expressions. This implies that when model data is used in a modifier, that it will use the model data as it is at the start of the scenario. Any updates to the model data during the scenario steps do not affect the possible choices for the example values. It is possible to use the argument value itself as one of the options. Using the actual argument as the only option (e.g. `:MOD: ${receiver}= [${receiver}]`) can force all other steps into a specific option, chosen directly from the scenario. For this to work, the single option must be a valid option for all steps using the same example value. @@ -199,6 +203,8 @@ In a then-step, modifiers behave slightly different. In then-steps no new option For now, variable data considers strict equivalence classes only. This means that all variants are considered equal for all purposes. If, for a certain scenario, a single valid example variant has been generated and executed, then this scenario is considered covered. There are no options yet to indicate deeper coverage targets based on data variations. It also implies that whenever any variant is valid, all scenario variants must be valid. And that regardless of which variant is chosen, the exact same scenarios can be chosen as the next one. This does however not mean that once a variant is chosen, that this variant will be used throughout the whole trace. If a scenario is selected multiple times in the same trace, then each occurrence will get new randomly selected data. +Modified example values do not cascade. If a modifier expression references another argument in its step that has a modifier of their own, then it will use the original example value, not the modified example value. + ## Configuration options ### Random seed diff --git a/atest/robotMBT tests/01__echo_suite/test_properties_are_retained.robot b/atest/robotMBT tests/01__echo_suite/test_properties_are_retained.robot new file mode 100644 index 00000000..5c06544f --- /dev/null +++ b/atest/robotMBT tests/01__echo_suite/test_properties_are_retained.robot @@ -0,0 +1,16 @@ +*** Settings *** +Documentation When reconstructing test cases it is expected that attributes belonging to the +... test case will remain untouched. +Suite Setup Treat this test suite Model-based +Test Tags my suite tag +Library robotmbt processor=echo + +*** Test Cases *** +Tags are retained + [Tags] my tag + Should Contain ${TEST TAGS} my tag + Should Contain ${TEST TAGS} my suite tag + +Documentation is retained + [Documentation] my documentation + Should Be Equal ${TEST DOCUMENTATION} my documentation diff --git a/pyproject.toml b/pyproject.toml index 012546da..d742728c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "robotframework-mbt" -version = "0.12.0" +version = "0.13.0" description = "Model-Based Testing in Robot framework with test case generation" readme = "README.md" authors = [{ name = "Johan Foederer", email = "github@famfoe.nl" }] diff --git a/robotmbt/modeller.py b/robotmbt/modeller.py index 8750121c..0d742005 100644 --- a/robotmbt/modeller.py +++ b/robotmbt/modeller.py @@ -72,36 +72,33 @@ def process_scenario(scenario: Scenario, model: ModelSpace) -> tuple[Scenario, S if 'error' in step.model_info: return None, None, dict(fail_masg=f"Error in scenario {scenario.name} " f"at step {step}: {step.model_info['error']}") - for expr in _relevant_expressions(step): - try: - if model.process_expression(expr, step.args) is False: - if step.gherkin_kw in ['when', None] and expr in step.model_info['OUT']: - part1, part2 = split_for_refinement(scenario, step) - return part1, part2, dict() - else: + if step.gherkin_kw is None and not step.model_info: + continue # model info is optional for action keywords + if 'IN' not in step.model_info or 'OUT' not in step.model_info: + raise Exception(f"Model info incomplete for step: {step}") + try: + if step.gherkin_kw in ['given', 'when', None]: + for expr in step.model_info['IN']: + if model.process_expression(expr, step.args) is False: return None, None, dict(fail_msg=f"Rejecting scenario {scenario.src_id}, " f"{scenario.name}, due to step '{step}': [{expr}] is False") - except TimeoutExceeded: - raise - except Exception as err: - return None, None, dict(fail_msg=f"Rejecting scenario {scenario.src_id}, " - f"{scenario.name}, due to step '{step}': [{expr}] {err}") + if step.gherkin_kw in ['when', 'then', None]: + for expr in step.model_info['OUT']: + if model.process_expression(expr, step.args) is False: + if step.gherkin_kw in ['when', None]: + part1, part2 = split_for_refinement(scenario, step) + return part1, part2, dict() + else: + return None, None, dict(fail_msg=f"Rejecting scenario {scenario.src_id}, " + f"{scenario.name}, due to step '{step}': [{expr}] is False") + except TimeoutExceeded: + raise + except Exception as err: + return None, None, dict(fail_msg=f"Rejecting scenario {scenario.src_id}, " + f"{scenario.name}, due to step '{step}': [{expr}] {err}") return scenario.copy(), None, dict() -def _relevant_expressions(step: Step) -> list[str]: - if step.gherkin_kw is None and not step.model_info: - return [] # model info is optional for action keywords - expressions = [] - if 'IN' not in step.model_info or 'OUT' not in step.model_info: - raise Exception(f"Model info incomplete for step: {step}") - if step.gherkin_kw in ['given', 'when', None]: - expressions += step.model_info['IN'] - if step.gherkin_kw in ['when', 'then', None]: - expressions += step.model_info['OUT'] - return expressions - - def split_for_refinement(scenario: Scenario, step: Step) -> tuple[Scenario, Scenario]: front, back = scenario.split_at_step(scenario.steps.index(step)) remaining_steps = '\n\t'.join([step.full_keyword, '- '*35] + [s.full_keyword for s in back.steps[1:]]) diff --git a/robotmbt/suitedata.py b/robotmbt/suitedata.py index 6bbb7d61..913f0f3b 100644 --- a/robotmbt/suitedata.py +++ b/robotmbt/suitedata.py @@ -71,11 +71,14 @@ def steps_with_errors(self): class Scenario: - def __init__(self, name: str, parent: Suite | None = None): + def __init__(self, name: str, parent: Suite, og_tc): self.name: str = name + # Keeping any keyword references in the copy of the original test case has a large + # performance impact. That is why some attributes are set to None. + self.og_tc = og_tc.copy(body=[], parent=None, _setup=None, _teardown=None) # Parent scenario is kept for easy searching, processing and referencing # after steps and scenarios have been potentially moved around - self.parent: Suite | None = parent + self.parent: Suite = parent self.setup: Step | None = None # Can be a single step or None self.teardown: Step | None = None # Can be a single step or None self.steps: list[Step] = [] diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index 0c9daacb..bc55e95a 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -30,7 +30,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from collections.abc import Callable +from collections.abc import Callable, Iterator from typing import Any import robot.model @@ -49,11 +49,13 @@ class SuiteReplacer: def __init__(self, processor: str = 'process_test_suite', processor_lib: str | None = None): self.current_suite: robot.model.TestSuite | None = None - self.robot_suite: robot.model.TestSuite | None = None + self.mbt_anchor_suite: robot.model.TestSuite | None = None self.processor_lib_name: str | None = processor_lib self.processor_name: str = processor self._processor_lib: SuiteProcessors | None | object = None self._processor_method: Callable[..., Suite] | None = None + self.suite_gen: list[Iterator[Suite]] = [] # Generator for on-the-fly suite insertion + self.test_case_gen: list[Iterator[Scenario]] = [] # Generator for on-the-fly test case insertion self.processor_options: dict[str, Any] = {} @property @@ -87,15 +89,16 @@ def treat_model_based(self, **kwargs): `Update model-based options`, then these arguments take precedence over the library option and affect only the current test suite. """ - self.robot_suite = self.current_suite - - logger.info(f"Analysing Robot test suite '{self.robot_suite.name}' for model-based execution.") + logger.info(f"Analysing Robot test suite '{self.current_suite.name}' for model-based execution.") local_settings = self.processor_options.copy() local_settings.update(kwargs) - master_suite = self.__process_robot_suite(self.robot_suite, parent=None) + master_suite = self.__process_robot_suite(self.current_suite, parent=None) modelbased_suite = self.processor_method(master_suite, **local_settings) - self.__clearTestSuite(self.robot_suite) - self.__generateRobotSuite(modelbased_suite, self.robot_suite) + self.suite_gen = [iter(modelbased_suite.suites)] + self.test_case_gen = [iter(modelbased_suite.scenarios)] + self.__clearTestSuite(self.current_suite) + self.add_next_new(self.current_suite) # add first test case only. Others are added at runtime by listeners. + self.mbt_anchor_suite = self.current_suite @keyword("Set model-based options") def set_model_based_options(self, **kwargs): @@ -139,7 +142,7 @@ def __process_robot_suite(self, in_suite: robot.model.TestSuite, parent: Suite | for st in in_suite.suites: out_suite.suites.append(self.__process_robot_suite(st, parent=out_suite)) for tc in in_suite.tests: - scenario = Scenario(tc.name, parent=out_suite) + scenario = Scenario(tc.name, parent=out_suite, og_tc=tc) if tc.setup: step_info = Step(tc.setup.name, *tc.setup.args, parent=scenario) step_info.add_robot_dependent_data(Robot._namespace.get_runner(step_info.org_step).keyword) @@ -170,38 +173,72 @@ def __clearTestSuite(self, suite: robot.model.TestSuite): suite.tests.clear() suite.suites.clear() - def __generateRobotSuite(self, suite_model: Suite, target_suite: robot.model.TestSuite): - for subsuite in suite_model.suites: - new_suite = target_suite.suites.create(name=subsuite.name) - new_suite.resource = target_suite.resource - if subsuite.setup: - new_suite.setup = rmodel.Keyword(name=subsuite.setup.keyword, - args=subsuite.setup.posnom_args_str, - type='setup') - if subsuite.teardown: - new_suite.teardown = rmodel.Keyword(name=subsuite.teardown.keyword, - args=subsuite.teardown.posnom_args_str, - type='teardown') - self.__generateRobotSuite(subsuite, new_suite) - for tc in suite_model.scenarios: - new_tc = target_suite.tests.create(name=tc.name) - if tc.setup: - new_tc.setup = rmodel.Keyword(name=tc.setup.keyword, - args=tc.setup.posnom_args_str, - type='setup') - if tc.teardown: - new_tc.teardown = rmodel.Keyword(name=tc.teardown.keyword, - args=tc.teardown.posnom_args_str, - type='teardown') - for step in tc.steps: - if step.keyword == 'VAR': - new_tc.body.create_var(step.posnom_args_str[0], step.posnom_args_str[1:]) - else: - new_tc.body.create_keyword(name=step.keyword, assign=step.assign, args=step.posnom_args_str) - - def _start_suite(self, suite: Suite | None, result): + def add_next_new(self, target_suite: robot.model.TestSuite): + """ + Adds the next-in-line test case to the current running test suite. When needed, + inserts all sub-suites leading up to the next test case. + """ + try: + new_suite = next(self.suite_gen[-1]) + self.suite_gen.append(iter(new_suite.suites)) + self.test_case_gen.append(iter(new_suite.scenarios)) + new_target = self.add_suite(new_suite, target_suite) + self.add_next_new(new_target) + except StopIteration: + try: + self.add_test(next(self.test_case_gen[-1]), target_suite) + except StopIteration: + pass + + @staticmethod + def add_suite(subsuite: Suite, target_suite: robot.model.TestSuite) -> robot.model.TestSuite: + new_suite = target_suite.suites.create(name=subsuite.name) + new_suite.resource = target_suite.resource + if subsuite.setup: + new_suite.setup = rmodel.Keyword(name=subsuite.setup.keyword, + args=subsuite.setup.posnom_args_str, + type='setup') + if subsuite.teardown: + new_suite.teardown = rmodel.Keyword(name=subsuite.teardown.keyword, + args=subsuite.teardown.posnom_args_str, + type='teardown') + return new_suite + + @staticmethod + def add_test(tc: Scenario, target_suite: robot.model.TestSuite): + new_tc = tc.og_tc.copy(name=tc.name, body=[]) + if tc.setup: + new_tc.setup = rmodel.Keyword(name=tc.setup.keyword, + args=tc.setup.posnom_args_str, + type='setup') + if tc.teardown: + new_tc.teardown = rmodel.Keyword(name=tc.teardown.keyword, + args=tc.teardown.posnom_args_str, + type='teardown') + for step in tc.steps: + if step.keyword == 'VAR': + new_tc.body.create_var(step.posnom_args_str[0], step.posnom_args_str[1:]) + else: + new_tc.body.create_keyword(name=step.keyword, assign=step.assign, args=step.posnom_args_str) + target_suite.tests.append(new_tc) + + def _start_suite(self, suite: robot.model.TestSuite, result): self.current_suite = suite - def _end_suite(self, suite: Suite | None, result): - if suite == self.robot_suite: - self.robot_suite = None + def _end_suite(self, suite: robot.model.TestSuite, result): + if suite == self.mbt_anchor_suite: + self.mbt_anchor_suite = None + if not self.mbt_anchor_suite: + return + self.suite_gen.pop() + self.test_case_gen.pop() + self.current_suite = self.current_suite.parent + self.add_next_new(self.current_suite) + + def _end_test(self, test_case: robot.model.TestCase, result): + if not self.mbt_anchor_suite: + return + try: + self.add_test(next(self.test_case_gen[-1]), self.current_suite) + except StopIteration: + pass diff --git a/robotmbt/version.py b/robotmbt/version.py index 69e84e16..0987cba6 100644 --- a/robotmbt/version.py +++ b/robotmbt/version.py @@ -1 +1 @@ -VERSION: str = '0.12.0' +VERSION: str = '0.13.0' diff --git a/utest/test_modeller.py b/utest/test_modeller.py new file mode 100644 index 00000000..c4c7a5aa --- /dev/null +++ b/utest/test_modeller.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# BSD 3-Clause License +# +# Copyright (c) 2026, J. Foederer +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from robotmbt.modeller import process_scenario + +from robotmbt.modelspace import ModelSpace + + +class TestModeller(unittest.TestCase): + def test_successful_scenario_evaluation(self): + scenario = ScenarioStub("my scenario") + scenario.steps.append(StepStub('initialise x', dict(IN=[], OUT=['new model', 'model.x = 0']))) + scenario.steps.append(StepStub('x should be 0', dict(IN=['model.x == 0'], OUT=[]))) + part1, part2, fail_info = process_scenario(scenario, ModelSpace()) + self.assertEqual(fail_info, {}) + self.assertEqual(part1.name, "my scenario") + self.assertIsNone(part2) + + def test_scenario_is_rejected_when_condition_evaluates_to_false(self): + scenario = ScenarioStub() + scenario.steps.append(StepStub('initialise x', dict(IN=[], OUT=['new model', 'model.x = 0']))) + scenario.steps.append(StepStub('x should be 2', dict(IN=['model.x == 2'], OUT=[]))) + part1, part2, fail_info = process_scenario(scenario, ModelSpace()) + self.assertIn("[model.x == 2] is False", fail_info['fail_msg']) + + def test_duplicate_in_out_condition_does_not_refine(self): + """ + Covers a prior defect where refinement would be entered unintended when the exact expression text + from an IN-condition that should fail the step, also appeared as an OUT-condition. + """ + scenario = ScenarioStub() + scenario.steps.append(StepStub('initialise x', dict(IN=[], OUT=['new model', 'model.x = 0']))) + scenario.steps.append(StepStub('x should be 2', dict(IN=['model.x == 2'], OUT=['model.x == 2']))) + part1, part2, fail_info = process_scenario(scenario, ModelSpace()) + self.assertIsNone(part2) + self.assertIn("[model.x == 2] is False", fail_info['fail_msg']) + + +class ScenarioStub: + def __init__(self, name: str = 'dummy'): + self.name = name + self.src_id = 0 + self.steps = [] + + def copy(self): + return ScenarioStub(self.name) + + +class StepStub: + def __init__(self, steptext: str, model_info: dict = {}) -> None: + self.org_step: str = steptext + first_word = steptext.split()[0].lower() + self.gherkin_kw = first_word if first_word in ['given', 'when', 'then'] else None + self.args = ArgStub() + self.model_info = model_info + + def __str__(self): + return self.org_step + + +class ArgStub(list): + def fill_in_args(self, text, as_code=True): + return text diff --git a/utest/test_suitedata.py b/utest/test_suitedata.py index 2623703a..691f339f 100644 --- a/utest/test_suitedata.py +++ b/utest/test_suitedata.py @@ -166,7 +166,7 @@ def setUp(self): @staticmethod def create_scenario(name, parent=None): - scenario = Scenario(name, parent) + scenario = Scenario(name, parent, RobotTestCaseStub()) scenario.steps = TestSteps.create_steps(parent=scenario) return scenario @@ -176,7 +176,7 @@ def test_longname_without_parent_is_just_the_name(self): def test_longname_with_parent_includes_both_names(self): def p(): return None # Create an object to assign the name attribute to p.longname = 'long' - scenario = Scenario('name', p) + scenario = Scenario('name', p, RobotTestCaseStub()) self.assertEqual(scenario.longname, 'long.name') def test_no_errors_when_ok(self): @@ -491,6 +491,11 @@ def map(x, y, z): return ([], []) def __iter__(_): return iter([]) +class RobotTestCaseStub: + def copy(self, **kwargs): + pass + + class StubStepArguments(list): modified = True # trigger modified status to get arguments processed, rather then just echoed