Java regex flags: "s", "d" and "U"

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt

@@ -442,7 +442,7 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
         }
 
         if(ctx.DOT() != null){
-            return VisitResult(AnyCharacterRxGene())
+            return VisitResult(AnyCharacterRxGene(currentFlags))
         }
 
         if(ctx.characterClass() != null){

core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt

@@ -15,24 +15,50 @@ import org.evomaster.core.search.service.Randomness
 import org.evomaster.core.search.service.mutator.MutationWeightControl
 import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo
 import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy
+import org.evomaster.core.utils.CharacterRange
+import org.evomaster.core.utils.MultiCharacterRange
+import org.evomaster.core.utils.RegexFlags
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
+private const val firstSurrogateChar = '\uD800'
+private const val lastSurrogateChar = '\uDFFF'
+private const val defaultLineTerminators = "\n\r\u0085\u2028\u2029"
+private const val unixLinesModeLineTerminators = "\n"
 
-class AnyCharacterRxGene : RxAtom, SimpleGene("."){
+class AnyCharacterRxGene(
+    val flags: RegexFlags = RegexFlags()
+) : RxAtom, SimpleGene(".") {
 
     companion object{
         private val log : Logger = LoggerFactory.getLogger(AnyCharacterRxGene::class.java)
+
+        /** All characters except for line terminators are recognized by "." in regex, see:
+         * https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#COMMENTS:~:text=range%20forming%20metacharacter.-,Line%20terminators,-A%20line%20terminator
+         */
+        /*
+         TODO in Java regex lone surrogates match ".", but this causes trouble when appearing in a rest path,
+            so for now we avoid surrogates altogether
+         */
+        val dotAllValidRanges = MultiCharacterRange(true,listOf(CharacterRange(firstSurrogateChar,lastSurrogateChar))) // all characters accepted
+        val defaultValidRanges = MultiCharacterRange(true,defaultLineTerminators).intersect(dotAllValidRanges)
+        val unixLinesValidRanges = MultiCharacterRange(true, unixLinesModeLineTerminators).intersect(dotAllValidRanges)
     }
 
     var value: Char = 'a'
 
+    val validRanges = when {
+        flags.dotAll -> dotAllValidRanges
+        flags.unixLines -> unixLinesValidRanges
+        else -> defaultValidRanges
+    }
+
     override fun checkForLocallyValidIgnoringChildren() : Boolean{
         return true
     }
 
     override fun copyContent(): Gene {
-        val copy = AnyCharacterRxGene()
+        val copy = AnyCharacterRxGene(flags)
         copy.value = this.value
         copy.name = this.name //in case name is changed from its default
         return copy
@@ -46,8 +72,11 @@ class AnyCharacterRxGene : RxAtom, SimpleGene("."){
     }
 
     override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) {
-        //TODO properly... this is just a tmp hack
-        value = randomness.nextWordChar()
+        val previous = value
+
+        while(tryToForceNewValue && previous == value) {
+            value = validRanges.sample(randomness)
+        }
     }
 
     override fun shallowMutate(randomness: Randomness, apc: AdaptiveParameterControl, mwc: MutationWeightControl, selectionStrategy: SubsetGeneMutationSelectionStrategy, enableAdaptiveGeneMutation: Boolean, additionalGeneMutationInfo: AdditionalGeneMutationInfo?): Boolean {

core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt

@@ -17,6 +17,11 @@ import org.evomaster.core.utils.UnicodeCache
 import org.slf4j.LoggerFactory
 import kotlin.collections.contains
 
+private const val firstASCIIChar = 0
+private const val lastASCIIChar = 0x7f
+private val unicodeCharClassModePredefinedEscapes = setOf('w', 'W', 'd', 'D', 's', 'S')
+private val posixEscapePrefix = setOf('p', 'P')
+
 /*
 \w	Find a word character
 \W	Find a non-word character
@@ -65,20 +70,20 @@ class CharacterClassEscapeRxGene(
         private val nonVerticalSpaceMultiCharRange = MultiCharacterRange(true, verticalSpaceSet)
 
         private val posixAsciiMultiCharRange: Map<String, MultiCharacterRange> = mapOf(
-                "Lower"  to listOf(CharacterRange('a', 'z')),
-                "Upper"  to listOf(CharacterRange('A', 'Z')),
-                "ASCII"  to listOf(CharacterRange(0, 0x7f)),
-                "Alpha"  to asciiLetterSet,
-                "Digit"  to digitSet,
-                "Alnum"  to digitSet + asciiLetterSet,
-                "Punct"  to punctuationSet,
-                "Graph"  to digitSet + asciiLetterSet + punctuationSet,
-                "Print"  to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"),
-                "Blank"  to stringToListOfCharacterRanges(" \t"),
-                "Cntrl"  to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"),
-                "XDigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')),
-                "Space"  to spaceSet,
-            )
+            "Lower"  to listOf(CharacterRange('a', 'z')),
+            "Upper"  to listOf(CharacterRange('A', 'Z')),
+            "ASCII"  to listOf(CharacterRange(firstASCIIChar, lastASCIIChar)),
+            "Alpha"  to asciiLetterSet,
+            "Digit"  to digitSet,
+            "Alnum"  to digitSet + asciiLetterSet,
+            "Punct"  to punctuationSet,
+            "Graph"  to digitSet + asciiLetterSet + punctuationSet,
+            "Print"  to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"),
+            "Blank"  to stringToListOfCharacterRanges(" \t"),
+            "Cntrl"  to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"),
+            "XDigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')),
+            "Space"  to spaceSet,
+        )
             // create both normal and negated version for all
             .flatMap { (key, value) ->
                 listOf(
@@ -106,29 +111,49 @@ class CharacterClassEscapeRxGene(
             throw IllegalArgumentException("Invalid type: $type")
         }
 
-        multiCharRange = when(type[0]){
-            'w' -> wordMultiCharRange
-            'W' -> nonWordMultiCharRange
-            'd' -> digitMultiCharRange
-            'D' -> nonDigitMultiCharRange
-            's' -> spaceMultiCharRange
-            'S' -> nonSpaceMultiCharRange
-            'v' -> verticalSpaceMultiCharRange
-            'V' -> nonVerticalSpaceMultiCharRange
-            'h' -> horizontalSpaceMultiCharRange
-            'H' -> nonHorizontalSpaceMultiCharRange
-            'p', 'P' -> {
-                val pLabel = type.substring(2, type.length - 1)
-                val negated = type[0].isUpperCase()
-                val lookupKey = if (negated) "^$pLabel" else pLabel
-                if (lookupKey !in posixAsciiMultiCharRange) {
-                    unicodeCache.getRanges(pLabel, negated)
-                } else {
-                    posixAsciiMultiCharRange[lookupKey]!!
+        val lowercasePosixKeys = posixAsciiMultiCharRange.keys.map{ it.lowercase() }
+
+        multiCharRange = if ( flags.unicodeCharacterClass &&
+            (type[0] in unicodeCharClassModePredefinedEscapes ||
+                    (type[0] in posixEscapePrefix && type.substring(2, type.length - 1).lowercase() in lowercasePosixKeys)) ) {
+            // UNICODE_CHARACTER_CLASSES flag is on, so these should now be in conformance with the recommendation of
+            // Annex C: Compatibility Properties of Unicode Regular Expression, see:
+            // https://www.unicode.org/reports/tr18/#Compatibility_Properties
+            val cacheLabel = when (type[0]) {
+                'd', 'D', 's', 'S', 'w', 'W' -> type.lowercase()
+                'p', 'P' -> type.substring(2, type.length - 1)
+                else -> //this should never happen due to check in init
+                    throw IllegalStateException("Type '\\$type' not supported yet")
+            }
+            unicodeCache.getRanges(cacheLabel, type[0].isUpperCase())
+        } else {
+            // regular predefined character classes
+            when(type[0]){
+                'w' -> wordMultiCharRange
+                'W' -> nonWordMultiCharRange
+                'd' -> digitMultiCharRange
+                'D' -> nonDigitMultiCharRange
+                's' -> spaceMultiCharRange
+                'S' -> nonSpaceMultiCharRange
+                'v' -> verticalSpaceMultiCharRange
+                'V' -> nonVerticalSpaceMultiCharRange
+                'h' -> horizontalSpaceMultiCharRange
+                'H' -> nonHorizontalSpaceMultiCharRange
+                'p', 'P' -> {
+                    val pLabel = type.substring(2, type.length - 1)
+                    val negated = type[0].isUpperCase()
+                    val lookupKey = if (negated) "^$pLabel" else pLabel
+                    if (lookupKey in posixAsciiMultiCharRange) {
+                        posixAsciiMultiCharRange[lookupKey]!!
+                    } else if (lookupKey.lowercase() in lowercasePosixKeys) {
+                        throw IllegalStateException("This escape (\\$type) is only valid when the \"U\" flag is on.")
+                    } else {
+                        unicodeCache.getRanges(pLabel, negated)
+                    }
                 }
+                else -> //this should never happen due to check in init
+                    throw IllegalStateException("Type '\\$type' not supported yet")
             }
-            else -> //this should never happen due to check in init
-                throw IllegalStateException("Type '\\$type' not supported yet")
         }
     }
 

core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt

@@ -200,6 +200,10 @@ class MultiCharacterRange internal constructor(val ranges: List<CharacterRange>)
         throw IllegalStateException("Cannot sample characters from an empty char range")
     }
 
+    fun intersect(other: MultiCharacterRange): MultiCharacterRange {
+        return intersect(this, other)
+    }
+
     val isEmpty: Boolean get() = ranges.isEmpty()
     val isNotEmpty: Boolean get() = ranges.isNotEmpty()
     val size: Int get() = ranges.size

core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt

@@ -30,15 +30,12 @@ data class ParsedFlagExpression(
     private val validFlagCharacters = setOf('i', 'u', 's', 'm', 'd', 'U', 'x')
 
 data class RegexFlags(
-    // currently implemented
     val caseInsensitive: Boolean = false,        // i
     val unicodeCase: Boolean = false,            // u, this flags modifies behaviour of "i" flag
-
-    // recognised but not yet implemented, validate() throws on these
     val dotAll: Boolean = false,                 // s
-    val multiline: Boolean = false,              // m
     val unixLines: Boolean = false,              // d
     val unicodeCharacterClass: Boolean = false,  // U
+    val multiline: Boolean = false,              // m
     val comments: Boolean = false,               // x
 ) {
 
@@ -135,10 +132,7 @@ data class RegexFlags(
      * Call this after merging, before recursing into the flagged disjunction.
      */
     fun validate() {
-        if (dotAll)                throw IllegalStateException("Regex flag 's' (DOTALL) is not yet supported")
-        if (multiline)             throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported")
-        if (unixLines)             throw IllegalStateException("Regex flag 'd' (UNIX_LINES) is not yet supported")
-        if (unicodeCharacterClass) throw IllegalStateException("Regex flag 'U' (UNICODE_CHARACTER_CLASS) is not yet supported")
+        if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported")
         if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported")
     }
 
@@ -147,7 +141,8 @@ data class RegexFlags(
      * and unicodeCase flag values.
      */
     fun isCaseable(codePoint: Int): Boolean {
-        return if (caseInsensitive && unicodeCase) {
+        // unicodeCharacterClass implies also unicodeCase
+        return if (caseInsensitive && (unicodeCase || unicodeCharacterClass)) {
             Character.toUpperCase(codePoint) != Character.toLowerCase(codePoint)
         }
         else if (caseInsensitive) {

core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt

@@ -182,6 +182,53 @@ class UnicodeCache {
         "javaUpperCase" to { cp -> Character.isUpperCase(cp) },
     )
 
+    // POSIX ESCAPES IN UNICODE CHAR CLASS MODE, case-insensitive, no prefix
+    private val unicodeCharClassModePredicates: Map<String, (Int) -> Boolean> = mapOf(
+        "w" to { cp -> getPredicate("Isalphabetic")(cp)
+                || getPredicate("Isdigit")(cp)
+                || getPredicate("Isjoincontrol")(cp)
+                || getPredicate("gc=M")(cp)
+        },
+        "d" to getPredicate("Isdigit"),
+        "s" to getPredicate("Iswhitespace"),
+
+
+        "lower" to getPredicate("Islowercase"),
+        "upper" to getPredicate("Isuppercase"),
+        "ascii" to { cp -> cp in 0..0x7f },
+        "alpha" to getPredicate("Isalphabetic"),
+        "digit" to getPredicate("Isdigit"),
+
+        "alnum" to { cp -> getPredicate("Isalphabetic")(cp)
+                || getPredicate("Isdigit")(cp) },
+
+        "punct" to getPredicate("Ispunctuation"),
+
+        "graph" to { cp -> !(getPredicate("Iswhitespace")(cp)
+                || getPredicate("gc=Cc")(cp)
+                || getPredicate("gc=Cs")(cp)
+                || getPredicate("gc=Cn")(cp))
+        },
+
+        "blank" to { cp -> getPredicate("Iswhitespace")(cp)
+                && !( getPredicate("gc=Zl")(cp)
+                || getPredicate("gc=Zp")(cp)
+                || cp in 0x0a..0x0d || cp == 0x85 )
+        },
+
+        "cntrl" to getPredicate("gc=Cc"),
+
+        "print" to { cp -> (getPredicate("graph")(cp)
+                || getPredicate("blank")(cp))
+                && !getPredicate("cntrl")(cp)
+        },
+
+        "xdigit" to { cp -> getPredicate("gc=Nd")(cp)
+                || getPredicate("Ishexdigit")(cp) },
+
+        "space" to getPredicate("Iswhitespace"),
+    )
+
     /*
        We need to normalize keys so that "gc=L" and "L" point to the same thing to avoid unnecessary re-computation.
        Keywords are always case-insensitive, prefixes are always case-sensitive. Most properties need either a keyword
@@ -237,8 +284,14 @@ class UnicodeCache {
             return pEscapeLabel
         }
 
+        // Unicode character class mode: exact match, case-insensitive
+        val unicodeModeKey = pEscapeLabel.lowercase()
+        if (unicodeModeKey in unicodeCharClassModePredicates) {
+            return unicodeModeKey
+        }
+
         // no match found
-        throw IllegalArgumentException("Unsupported/illegal category, binary property or java method")
+        throw IllegalArgumentException("Unsupported/illegal category, binary property or java method: $pEscapeLabel")
     }
 
     /*
@@ -292,18 +345,11 @@ class UnicodeCache {
             key
         }
 
-        val predicate = when (key) {
-            in scriptPredicates -> scriptPredicates[key]
-            in blockPredicates -> blockPredicates[key]
-            in generalCategoryPredicates -> generalCategoryPredicates[key]
-            in binaryPropertiesPredicates -> binaryPropertiesPredicates[key]
-            in javaCharacterMethodPredicates -> javaCharacterMethodPredicates[key]
-            else -> null
-        }
+        val predicate = getPredicate(key)
 
         // first we compute and cache the base key (non-negated)
         cache.computeIfAbsent(key) {
-            computeRanges(key, predicate!!)
+            computeRanges(key, predicate)
         }
 
         // if the base kay was requested just return
@@ -314,4 +360,14 @@ class UnicodeCache {
             MultiCharacterRange(true, cache[key]!!.ranges)
         }
     }
+
+    private fun getPredicate(key: String): ((Int) -> Boolean) = when (key) {
+        in scriptPredicates -> scriptPredicates[key]!!
+        in blockPredicates -> blockPredicates[key]!!
+        in generalCategoryPredicates -> generalCategoryPredicates[key]!!
+        in binaryPropertiesPredicates -> binaryPropertiesPredicates[key]!!
+        in javaCharacterMethodPredicates -> javaCharacterMethodPredicates[key]!!
+        in unicodeCharClassModePredicates -> unicodeCharClassModePredicates[key]!!
+        else -> throw IllegalArgumentException("Unsupported/illegal category, binary property or java method")
+    }
 }