Config skip fields

core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java

@@ -625,6 +625,10 @@ protected void assertTextInTests(String outputFolder, String className, String c
         assertTextInTests(outputFolder,className, l -> l.contains(content));
     }
 
+    protected void assertTextNotInTests(String outputFolder, String className, String content) {
+        assertTextNotInTests(outputFolder, className, l -> l.contains(content));
+    }
+
         /**
          * assert a certain text in the generated tests
          * @param outputFolder the folder where the test is
@@ -643,6 +647,24 @@ protected void assertTextInTests(String outputFolder, String className, Predicat
         }
     }
 
+    /**
+     * Assert that a certain text is not present in the generated tests.
+     * @param outputFolder the folder where the test is
+     * @param className the complete test name
+     * @param condition is the content to check
+     */
+    protected void assertTextNotInTests(String outputFolder, String className, Predicate<String> condition) {
+        String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt";
+        Path test = Paths.get(path);
+        try {
+            boolean ok = Files.lines(test).noneMatch(condition);
+            String msg = "Found line with forbidden condition in "+className+" in "+outputFolder;
+            assertTrue(ok, msg);
+        }catch (IOException e){
+            throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage());
+        }
+    }
+
     protected List<String> getFunctionNames(String outputFolder, String className){
 
         //this specific on how we create tests

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectSkipFieldEMTest.kt

@@ -0,0 +1,45 @@
+package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect
+
+import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController
+import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+
+class FlakinessDetectSkipFieldEMTest : SpringTestBase() {
+
+    companion object {
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(FlakinessDetectController())
+        }
+    }
+
+    @Test
+    fun testRunEMWithSkippedFlakyField() {
+
+        val outputFolder = "FlakinessDetectSkippedFieldEM"
+        val outputClass = "org.foo.FlakinessDetectSkippedFieldEM"
+
+        runTestHandlingFlakyAndCompilation(
+            outputFolder,
+            outputClass,
+            30,
+            true
+        ) { args: MutableList<String> ->
+
+            setOption(args, "minimize", "true")
+            setOption(args, "handleFlakiness", "false")
+            setOption(args, "endpointFocus", "/api/flakinessdetect/multiexecution")
+            setOption(args, "fieldsToSkipInAssertions", "first,second")
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.isNotEmpty())
+            assertTextInTests(outputFolder, outputClass, ".body(\"'stable'\"")
+            assertTextNotInTests(outputFolder, outputClass, ".body(\"'first'\"")
+            assertTextNotInTests(outputFolder, outputClass, ".body(\"'second'\"")
+        }
+    }
+}

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -2039,6 +2039,10 @@ class EMConfig {
             "NOTE: this should not cause any tests to fail.")
     var enableBasicAssertions = true
 
+    @Experimental
+    @Cfg("Comma-separated list of response field names to skip when generating assertions.")
+    var fieldsToSkipInAssertions = ""
+
     @Cfg("Apply method replacement heuristics to smooth the search landscape." +
             " Note that the method replacement instrumentations would still be applied, it is just that their testing targets" +
             " will be ignored in the fitness function if this option is set to false.")

core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt

@@ -658,17 +658,26 @@ abstract class ApiTestCaseWriter : TestCaseWriter() {
     /**
      * Some fields might lead to flackiness, eg assertions on timestamps.
      */
-    protected fun isFieldToSkip(fieldName: String) =
-    //TODO this should be from EMConfig
-            /*
-                There are some fields like "id" which are often non-deterministic,
-                which unfortunately would lead to flaky tests
-            */
-            listOf(
-                    "id",
-                    "timestamp", //needed since timestamps will change between runs
-                    "self" //TODO: temporary hack. Needed since ports might change between runs.
-            ).contains(fieldName.lowercase())
+    protected fun isFieldToSkip(fieldName: String): Boolean {
+        val field = fieldName.lowercase()
+
+        /*
+            There are some fields like "id" which are often non-deterministic,
+            which unfortunately would lead to flaky tests
+        */
+        val defaultFieldsToSkip = listOf(
+                "id",
+                "timestamp", //needed since timestamps will change between runs
+                "self" //TODO: temporary hack. Needed since ports might change between runs.
+        )
+
+        val configuredFieldsToSkip = config.fieldsToSkipInAssertions
+                .split(",")
+                .map { it.trim().lowercase() }
+                .filter { it.isNotEmpty() }
+
+        return defaultFieldsToSkip.contains(field) || configuredFieldsToSkip.contains(field)
+    }
 
     /**
      * Some content may be lead to problems in the resultant test case.

docs/options.md

@@ -294,6 +294,7 @@ There are 3 types of options:
 |`externalServiceIP`| __String__. User provided external service IP. When EvoMaster mocks external services, mock server instances will run on local addresses starting from this provided address. Min value is 127.0.0.4. Lower values like 127.0.0.2 and 127.0.0.3 are reserved. *Constraints*: `regex (?!^0*127(\.0*0){2}\.0*[0123]$)^0*127(\.0*(25[0-5]\|2[0-4][0-9]\|1?[0-9]?[0-9])){3}$`. *Default value*: `127.0.0.4`.|
 |`externalServiceIPSelectionStrategy`| __Enum__. Specify a method to select the first external service spoof IP address. *Valid values*: `NONE, DEFAULT, USER, RANDOM`. *Default value*: `NONE`.|
 |`extractRedisExecutionInfo`| __Boolean__. Enable extracting Redis execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.|
+|`fieldsToSkipInAssertions`| __String__. Comma-separated list of response field names to skip when generating assertions. *Default value*: `""`.|
 |`generateRedisData`| __Boolean__. Enable EvoMaster to generate Redis data with direct accesses to the database. *Depends on*: `blackBox=false`. *Default value*: `false`.|
 |`generateSqlDataWithDSE`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use Dynamic Symbolic Execution. *Depends on*: `blackBox=false`. *Default value*: `false`.|
 |`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. Note that flakiness is now supported only for fuzzing REST APIs. *Default value*: `false`.|