From 9652c46198b17f5feeffce1e4247d4b9f5cfed07 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Fri, 3 Jul 2026 16:21:53 +0800 Subject: [PATCH 1/4] add config --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 4 ++++ docs/options.md | 1 + 2 files changed, 5 insertions(+) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 09b1dff31a..581e1450a4 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2039,6 +2039,10 @@ class EMConfig { "NOTE: this should not cause any tests to fail.") var enableBasicAssertions = true + @Experimental + @Cfg("Comma-separated list of response field names to skip when generating assertions.") + var fieldsToSkipInAssertions = "" + @Cfg("Apply method replacement heuristics to smooth the search landscape." + " Note that the method replacement instrumentations would still be applied, it is just that their testing targets" + " will be ignored in the fitness function if this option is set to false.") diff --git a/docs/options.md b/docs/options.md index e870a9a1c3..bb61ec83aa 100644 --- a/docs/options.md +++ b/docs/options.md @@ -294,6 +294,7 @@ There are 3 types of options: |`externalServiceIP`| __String__. User provided external service IP. When EvoMaster mocks external services, mock server instances will run on local addresses starting from this provided address. Min value is 127.0.0.4. Lower values like 127.0.0.2 and 127.0.0.3 are reserved. *Constraints*: `regex (?!^0*127(\.0*0){2}\.0*[0123]$)^0*127(\.0*(25[0-5]\|2[0-4][0-9]\|1?[0-9]?[0-9])){3}$`. *Default value*: `127.0.0.4`.| |`externalServiceIPSelectionStrategy`| __Enum__. Specify a method to select the first external service spoof IP address. *Valid values*: `NONE, DEFAULT, USER, RANDOM`. *Default value*: `NONE`.| |`extractRedisExecutionInfo`| __Boolean__. Enable extracting Redis execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.| +|`fieldsToSkipInAssertions`| __String__. Comma-separated list of response field names to skip when generating assertions. *Default value*: `""`.| |`generateRedisData`| __Boolean__. Enable EvoMaster to generate Redis data with direct accesses to the database. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`generateSqlDataWithDSE`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use Dynamic Symbolic Execution. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. Note that flakiness is now supported only for fuzzing REST APIs. *Default value*: `false`.| From 6e7912efc14a34f808b5c7d43320efc5d9bc8541 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Fri, 3 Jul 2026 16:24:33 +0800 Subject: [PATCH 2/4] isFieldToSkip --- .../core/output/service/ApiTestCaseWriter.kt | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 1c09a332e3..85287f4f40 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -658,17 +658,26 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { /** * Some fields might lead to flackiness, eg assertions on timestamps. */ - protected fun isFieldToSkip(fieldName: String) = - //TODO this should be from EMConfig - /* - There are some fields like "id" which are often non-deterministic, - which unfortunately would lead to flaky tests - */ - listOf( - "id", - "timestamp", //needed since timestamps will change between runs - "self" //TODO: temporary hack. Needed since ports might change between runs. - ).contains(fieldName.lowercase()) + protected fun isFieldToSkip(fieldName: String): Boolean { + val field = fieldName.lowercase() + + /* + There are some fields like "id" which are often non-deterministic, + which unfortunately would lead to flaky tests + */ + val defaultFieldsToSkip = listOf( + "id", + "timestamp", //needed since timestamps will change between runs + "self" //TODO: temporary hack. Needed since ports might change between runs. + ) + + val configuredFieldsToSkip = config.fieldsToSkipInAssertions + .split(",") + .map { it.trim().lowercase() } + .filter { it.isNotEmpty() } + + return defaultFieldsToSkip.contains(field) || configuredFieldsToSkip.contains(field) + } /** * Some content may be lead to problems in the resultant test case. From 5af25f4d6c8e1cb97af30472a830d720be297cd4 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Fri, 3 Jul 2026 16:36:54 +0800 Subject: [PATCH 3/4] add assertTextNotInTests --- .../e2etests/utils/EnterpriseTestBase.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 6bfe262a04..50a48b47f1 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -625,6 +625,10 @@ protected void assertTextInTests(String outputFolder, String className, String c assertTextInTests(outputFolder,className, l -> l.contains(content)); } + protected void assertTextNotInTests(String outputFolder, String className, String content) { + assertTextNotInTests(outputFolder, className, l -> l.contains(content)); + } + /** * assert a certain text in the generated tests * @param outputFolder the folder where the test is @@ -643,6 +647,24 @@ protected void assertTextInTests(String outputFolder, String className, Predicat } } + /** + * Assert that a certain text is not present in the generated tests. + * @param outputFolder the folder where the test is + * @param className the complete test name + * @param condition is the content to check + */ + protected void assertTextNotInTests(String outputFolder, String className, Predicate condition) { + String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt"; + Path test = Paths.get(path); + try { + boolean ok = Files.lines(test).noneMatch(condition); + String msg = "Found line with forbidden condition in "+className+" in "+outputFolder; + assertTrue(ok, msg); + }catch (IOException e){ + throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); + } + } + protected List getFunctionNames(String outputFolder, String className){ //this specific on how we create tests From 536fc3eec1d7ae26c41a6567cb2d15875777b966 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Fri, 3 Jul 2026 16:38:20 +0800 Subject: [PATCH 4/4] add e2e test for flaky field --- .../FlakinessDetectSkipFieldEMTest.kt | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt new file mode 100644 index 0000000000..ce56248b5b --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FlakinessDetectSkipFieldEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FlakinessDetectController()) + } + } + + @Test + fun testRunEMWithSkippedFlakyField() { + + val outputFolder = "FlakinessDetectSkippedFieldEM" + val outputClass = "org.foo.FlakinessDetectSkippedFieldEM" + + runTestHandlingFlakyAndCompilation( + outputFolder, + outputClass, + 30, + true + ) { args: MutableList -> + + setOption(args, "minimize", "true") + setOption(args, "handleFlakiness", "false") + setOption(args, "endpointFocus", "/api/flakinessdetect/multiexecution") + setOption(args, "fieldsToSkipInAssertions", "first,second") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.isNotEmpty()) + assertTextInTests(outputFolder, outputClass, ".body(\"'stable'\"") + assertTextNotInTests(outputFolder, outputClass, ".body(\"'first'\"") + assertTextNotInTests(outputFolder, outputClass, ".body(\"'second'\"") + } + } +}