diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 6bfe262a04..50a48b47f1 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -625,6 +625,10 @@ protected void assertTextInTests(String outputFolder, String className, String c assertTextInTests(outputFolder,className, l -> l.contains(content)); } + protected void assertTextNotInTests(String outputFolder, String className, String content) { + assertTextNotInTests(outputFolder, className, l -> l.contains(content)); + } + /** * assert a certain text in the generated tests * @param outputFolder the folder where the test is @@ -643,6 +647,24 @@ protected void assertTextInTests(String outputFolder, String className, Predicat } } + /** + * Assert that a certain text is not present in the generated tests. + * @param outputFolder the folder where the test is + * @param className the complete test name + * @param condition is the content to check + */ + protected void assertTextNotInTests(String outputFolder, String className, Predicate condition) { + String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt"; + Path test = Paths.get(path); + try { + boolean ok = Files.lines(test).noneMatch(condition); + String msg = "Found line with forbidden condition in "+className+" in "+outputFolder; + assertTrue(ok, msg); + }catch (IOException e){ + throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); + } + } + protected List getFunctionNames(String outputFolder, String className){ //this specific on how we create tests diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt new file mode 100644 index 0000000000..ce56248b5b --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FlakinessDetectSkipFieldEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FlakinessDetectController()) + } + } + + @Test + fun testRunEMWithSkippedFlakyField() { + + val outputFolder = "FlakinessDetectSkippedFieldEM" + val outputClass = "org.foo.FlakinessDetectSkippedFieldEM" + + runTestHandlingFlakyAndCompilation( + outputFolder, + outputClass, + 30, + true + ) { args: MutableList -> + + setOption(args, "minimize", "true") + setOption(args, "handleFlakiness", "false") + setOption(args, "endpointFocus", "/api/flakinessdetect/multiexecution") + setOption(args, "fieldsToSkipInAssertions", "first,second") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.isNotEmpty()) + assertTextInTests(outputFolder, outputClass, ".body(\"'stable'\"") + assertTextNotInTests(outputFolder, outputClass, ".body(\"'first'\"") + assertTextNotInTests(outputFolder, outputClass, ".body(\"'second'\"") + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 09b1dff31a..581e1450a4 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2039,6 +2039,10 @@ class EMConfig { "NOTE: this should not cause any tests to fail.") var enableBasicAssertions = true + @Experimental + @Cfg("Comma-separated list of response field names to skip when generating assertions.") + var fieldsToSkipInAssertions = "" + @Cfg("Apply method replacement heuristics to smooth the search landscape." + " Note that the method replacement instrumentations would still be applied, it is just that their testing targets" + " will be ignored in the fitness function if this option is set to false.") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 1c09a332e3..85287f4f40 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -658,17 +658,26 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { /** * Some fields might lead to flackiness, eg assertions on timestamps. */ - protected fun isFieldToSkip(fieldName: String) = - //TODO this should be from EMConfig - /* - There are some fields like "id" which are often non-deterministic, - which unfortunately would lead to flaky tests - */ - listOf( - "id", - "timestamp", //needed since timestamps will change between runs - "self" //TODO: temporary hack. Needed since ports might change between runs. - ).contains(fieldName.lowercase()) + protected fun isFieldToSkip(fieldName: String): Boolean { + val field = fieldName.lowercase() + + /* + There are some fields like "id" which are often non-deterministic, + which unfortunately would lead to flaky tests + */ + val defaultFieldsToSkip = listOf( + "id", + "timestamp", //needed since timestamps will change between runs + "self" //TODO: temporary hack. Needed since ports might change between runs. + ) + + val configuredFieldsToSkip = config.fieldsToSkipInAssertions + .split(",") + .map { it.trim().lowercase() } + .filter { it.isNotEmpty() } + + return defaultFieldsToSkip.contains(field) || configuredFieldsToSkip.contains(field) + } /** * Some content may be lead to problems in the resultant test case. diff --git a/docs/options.md b/docs/options.md index e870a9a1c3..bb61ec83aa 100644 --- a/docs/options.md +++ b/docs/options.md @@ -294,6 +294,7 @@ There are 3 types of options: |`externalServiceIP`| __String__. User provided external service IP. When EvoMaster mocks external services, mock server instances will run on local addresses starting from this provided address. Min value is 127.0.0.4. Lower values like 127.0.0.2 and 127.0.0.3 are reserved. *Constraints*: `regex (?!^0*127(\.0*0){2}\.0*[0123]$)^0*127(\.0*(25[0-5]\|2[0-4][0-9]\|1?[0-9]?[0-9])){3}$`. *Default value*: `127.0.0.4`.| |`externalServiceIPSelectionStrategy`| __Enum__. Specify a method to select the first external service spoof IP address. *Valid values*: `NONE, DEFAULT, USER, RANDOM`. *Default value*: `NONE`.| |`extractRedisExecutionInfo`| __Boolean__. Enable extracting Redis execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.| +|`fieldsToSkipInAssertions`| __String__. Comma-separated list of response field names to skip when generating assertions. *Default value*: `""`.| |`generateRedisData`| __Boolean__. Enable EvoMaster to generate Redis data with direct accesses to the database. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`generateSqlDataWithDSE`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use Dynamic Symbolic Execution. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. Note that flakiness is now supported only for fuzzing REST APIs. *Default value*: `false`.|