From f9dec510f9542de595714adf140339ae2a299544 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 21:54:57 -0300 Subject: [PATCH 01/12] Made pLabels case insensitive for p escapes --- .../gene/regex/CharacterClassEscapeRxGene.kt | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt index 06797e7932..c1d95d2be2 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt @@ -65,20 +65,20 @@ class CharacterClassEscapeRxGene( private val nonVerticalSpaceMultiCharRange = MultiCharacterRange(true, verticalSpaceSet) private val posixAsciiMultiCharRange: Map = mapOf( - "Lower" to listOf(CharacterRange('a', 'z')), - "Upper" to listOf(CharacterRange('A', 'Z')), - "ASCII" to listOf(CharacterRange(0, 0x7f)), - "Alpha" to asciiLetterSet, - "Digit" to digitSet, - "Alnum" to digitSet + asciiLetterSet, - "Punct" to punctuationSet, - "Graph" to digitSet + asciiLetterSet + punctuationSet, - "Print" to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"), - "Blank" to stringToListOfCharacterRanges(" \t"), - "Cntrl" to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"), - "XDigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')), - "Space" to spaceSet, - ) + "lower" to listOf(CharacterRange('a', 'z')), + "upper" to listOf(CharacterRange('A', 'Z')), + "ascii" to listOf(CharacterRange(0, 0x7f)), + "alpha" to asciiLetterSet, + "digit" to digitSet, + "alnum" to digitSet + asciiLetterSet, + "punct" to punctuationSet, + "graph" to digitSet + asciiLetterSet + punctuationSet, + "print" to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"), + "blank" to stringToListOfCharacterRanges(" \t"), + "cntrl" to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"), + "xdigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')), + "space" to spaceSet, + ) // create both normal and negated version for all .flatMap { (key, value) -> listOf( @@ -120,11 +120,11 @@ class CharacterClassEscapeRxGene( 'p', 'P' -> { val pLabel = type.substring(2, type.length - 1) val negated = type[0].isUpperCase() - val lookupKey = if (negated) "^$pLabel" else pLabel - if (lookupKey !in posixAsciiMultiCharRange) { - unicodeCache.getRanges(pLabel, negated) - } else { + val lookupKey = (if (negated) "^$pLabel" else pLabel).lowercase() + if (lookupKey in posixAsciiMultiCharRange) { posixAsciiMultiCharRange[lookupKey]!! + } else { + unicodeCache.getRanges(pLabel, negated) } } else -> //this should never happen due to check in init From f5931ccc74fada9fac454a95276b16314474a954 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 22:24:50 -0300 Subject: [PATCH 02/12] Implemented AnyCharacterRxGene.randomize() properly. --- .../search/gene/regex/AnyCharacterRxGene.kt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 9a4aecb545..053e2e55bd 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -15,6 +15,7 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.utils.MultiCharacterRange import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -23,10 +24,17 @@ class AnyCharacterRxGene : RxAtom, SimpleGene("."){ companion object{ private val log : Logger = LoggerFactory.getLogger(AnyCharacterRxGene::class.java) + + /** All characters except for line terminators are recognized by "." in regex, see: + * https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#COMMENTS:~:text=range%20forming%20metacharacter.-,Line%20terminators,-A%20line%20terminator + */ + val defaultValidRanges = MultiCharacterRange(true,"\n\r\u0085\u2028\u2029") } var value: Char = 'a' + val validRanges = defaultValidRanges + override fun checkForLocallyValidIgnoringChildren() : Boolean{ return true } @@ -46,8 +54,13 @@ class AnyCharacterRxGene : RxAtom, SimpleGene("."){ } override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) { - //TODO properly... this is just a tmp hack - value = randomness.nextWordChar() + val previous = value + + value = validRanges.sample(randomness) + + if (previous == value) { + randomize(randomness, tryToForceNewValue) + } } override fun shallowMutate(randomness: Randomness, apc: AdaptiveParameterControl, mwc: MutationWeightControl, selectionStrategy: SubsetGeneMutationSelectionStrategy, enableAdaptiveGeneMutation: Boolean, additionalGeneMutationInfo: AdditionalGeneMutationInfo?): Boolean { From 8d42396eb46f188d438cfcacafd2bc565df98b68 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 22:34:29 -0300 Subject: [PATCH 03/12] Java regex added embedded "s" (DOTALL) flag. --- .../evomaster/core/parser/GeneRegexJavaVisitor.kt | 2 +- .../core/search/gene/regex/AnyCharacterRxGene.kt | 14 ++++++++++---- .../evomaster/core/utils/MultiCharacterRange.kt | 2 +- .../kotlin/org/evomaster/core/utils/RegexFlags.kt | 5 ++--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt index 8f3d784922..314cce1561 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt @@ -383,7 +383,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor(){ } if(ctx.DOT() != null){ - return VisitResult(AnyCharacterRxGene()) + return VisitResult(AnyCharacterRxGene(currentFlags)) } if(ctx.characterClass() != null){ diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 053e2e55bd..09f6ba51c6 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -16,11 +16,13 @@ import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy import org.evomaster.core.utils.MultiCharacterRange +import org.evomaster.core.utils.RegexFlags import org.slf4j.Logger import org.slf4j.LoggerFactory - -class AnyCharacterRxGene : RxAtom, SimpleGene("."){ +class AnyCharacterRxGene( + val flags: RegexFlags = RegexFlags() +) : RxAtom, SimpleGene(".") { companion object{ private val log : Logger = LoggerFactory.getLogger(AnyCharacterRxGene::class.java) @@ -29,18 +31,22 @@ class AnyCharacterRxGene : RxAtom, SimpleGene("."){ * https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#COMMENTS:~:text=range%20forming%20metacharacter.-,Line%20terminators,-A%20line%20terminator */ val defaultValidRanges = MultiCharacterRange(true,"\n\r\u0085\u2028\u2029") + val dotAllValidRanges = MultiCharacterRange(true, "") // all characters accepted } var value: Char = 'a' - val validRanges = defaultValidRanges + val validRanges = when { + flags.dotAll -> dotAllValidRanges + else -> defaultValidRanges + } override fun checkForLocallyValidIgnoringChildren() : Boolean{ return true } override fun copyContent(): Gene { - val copy = AnyCharacterRxGene() + val copy = AnyCharacterRxGene(flags) copy.value = this.value copy.name = this.name //in case name is changed from its default return copy diff --git a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt index 8ea9050bae..e708cf917a 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt @@ -17,7 +17,7 @@ class MultiCharacterRange internal constructor(val ranges: List) } operator fun invoke(negated: Boolean, ranges: List): MultiCharacterRange { - if (ranges.isEmpty()) { + if (ranges.isEmpty() && !negated) { throw IllegalArgumentException("No defined ranges") } diff --git a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt index 90f8408b36..6f58d6a563 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt @@ -31,9 +31,9 @@ data class RegexFlags( // currently implemented val caseInsensitive: Boolean = false, // i val unicodeCase: Boolean = false, // u, this flags modifies behaviour of "i" flag + val dotAll: Boolean = false, // s // recognised but not yet implemented, validate() throws on these - val dotAll: Boolean = false, // s val multiline: Boolean = false, // m val unixLines: Boolean = false, // d val unicodeCharacterClass: Boolean = false, // U @@ -117,11 +117,10 @@ data class RegexFlags( * Call this after merging, before recursing into the flagged disjunction. */ fun validate() { - if (dotAll) throw IllegalStateException("Regex flag 's' (DOTALL) is not yet supported") if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported") if (unixLines) throw IllegalStateException("Regex flag 'd' (UNIX_LINES) is not yet supported") if (unicodeCharacterClass) throw IllegalStateException("Regex flag 'U' (UNICODE_CHARACTER_CLASS) is not yet supported") - if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported") + if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported") } /** From 897b9e07ef6beb88c1b93770568569e6ea0f7fe3 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 22:37:34 -0300 Subject: [PATCH 04/12] Java regex added embedded "d" (UNIX_LINES) flag. --- .../org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt | 2 ++ core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 09f6ba51c6..2c4b9f3cc6 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -32,12 +32,14 @@ class AnyCharacterRxGene( */ val defaultValidRanges = MultiCharacterRange(true,"\n\r\u0085\u2028\u2029") val dotAllValidRanges = MultiCharacterRange(true, "") // all characters accepted + val unixLinesValidRanges = MultiCharacterRange(true, "\n") } var value: Char = 'a' val validRanges = when { flags.dotAll -> dotAllValidRanges + flags.unixLines -> unixLinesValidRanges else -> defaultValidRanges } diff --git a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt index 6f58d6a563..ddf292a8b9 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt @@ -32,10 +32,10 @@ data class RegexFlags( val caseInsensitive: Boolean = false, // i val unicodeCase: Boolean = false, // u, this flags modifies behaviour of "i" flag val dotAll: Boolean = false, // s + val unixLines: Boolean = false, // d // recognised but not yet implemented, validate() throws on these val multiline: Boolean = false, // m - val unixLines: Boolean = false, // d val unicodeCharacterClass: Boolean = false, // U val comments: Boolean = false, // x ) { @@ -118,7 +118,6 @@ data class RegexFlags( */ fun validate() { if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported") - if (unixLines) throw IllegalStateException("Regex flag 'd' (UNIX_LINES) is not yet supported") if (unicodeCharacterClass) throw IllegalStateException("Regex flag 'U' (UNICODE_CHARACTER_CLASS) is not yet supported") if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported") } From 1c22f64e0c1904cf8e000157311c4cdf0b82bf3b Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 22:46:20 -0300 Subject: [PATCH 05/12] Java regex added embedded "U" (UNICODE_CHARACTER_CLASS) flag. --- .../gene/regex/CharacterClassEscapeRxGene.kt | 58 ++++++++------ .../org/evomaster/core/utils/RegexFlags.kt | 7 +- .../org/evomaster/core/utils/UnicodeCache.kt | 75 ++++++++++++++++--- 3 files changed, 104 insertions(+), 36 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt index c1d95d2be2..7525d6c9cf 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt @@ -82,7 +82,7 @@ class CharacterClassEscapeRxGene( // create both normal and negated version for all .flatMap { (key, value) -> listOf( - key to MultiCharacterRange(value), + key to MultiCharacterRange(false, value), "^$key" to MultiCharacterRange(true, value) ) }.toMap() @@ -106,29 +106,43 @@ class CharacterClassEscapeRxGene( throw IllegalArgumentException("Invalid type: $type") } - multiCharRange = when(type[0]){ - 'w' -> wordMultiCharRange - 'W' -> nonWordMultiCharRange - 'd' -> digitMultiCharRange - 'D' -> nonDigitMultiCharRange - 's' -> spaceMultiCharRange - 'S' -> nonSpaceMultiCharRange - 'v' -> verticalSpaceMultiCharRange - 'V' -> nonVerticalSpaceMultiCharRange - 'h' -> horizontalSpaceMultiCharRange - 'H' -> nonHorizontalSpaceMultiCharRange - 'p', 'P' -> { - val pLabel = type.substring(2, type.length - 1) - val negated = type[0].isUpperCase() - val lookupKey = (if (negated) "^$pLabel" else pLabel).lowercase() - if (lookupKey in posixAsciiMultiCharRange) { - posixAsciiMultiCharRange[lookupKey]!! - } else { - unicodeCache.getRanges(pLabel, negated) + multiCharRange = if ( flags.unicodeCharacterClass && (type[0] in "wWdDsS" || (type[0] in "pP" && type.substring(2, type.length - 1).lowercase() in posixAsciiMultiCharRange)) ) { + // UNICODE_CHARACTER_CLASSES flag is on, so these should now be in conformance with the recommendation of + // Annex C: Compatibility Properties of Unicode Regular Expression, see: + // https://www.unicode.org/reports/tr18/#Compatibility_Properties + val cacheLabel = when (type[0]) { + 'd', 'D', 's', 'S', 'w', 'W' -> type.lowercase() + 'p', 'P' -> type.substring(2, type.length - 1) + else -> //this should never happen due to check in init + throw IllegalStateException("Type '\\$type' not supported yet") + } + unicodeCache.getRanges(cacheLabel, type[0].isUpperCase()) + } else { + // regular predefined character classes + when(type[0]){ + 'w' -> wordMultiCharRange + 'W' -> nonWordMultiCharRange + 'd' -> digitMultiCharRange + 'D' -> nonDigitMultiCharRange + 's' -> spaceMultiCharRange + 'S' -> nonSpaceMultiCharRange + 'v' -> verticalSpaceMultiCharRange + 'V' -> nonVerticalSpaceMultiCharRange + 'h' -> horizontalSpaceMultiCharRange + 'H' -> nonHorizontalSpaceMultiCharRange + 'p', 'P' -> { + val pLabel = type.substring(2, type.length - 1) + val negated = type[0].isUpperCase() + val lookupKey = (if (negated) "^$pLabel" else pLabel).lowercase() + if (lookupKey in posixAsciiMultiCharRange) { + posixAsciiMultiCharRange[lookupKey]!! + } else { + unicodeCache.getRanges(pLabel, negated) + } } + else -> //this should never happen due to check in init + throw IllegalStateException("Type '\\$type' not supported yet") } - else -> //this should never happen due to check in init - throw IllegalStateException("Type '\\$type' not supported yet") } } diff --git a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt index ddf292a8b9..a382924a8e 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt @@ -33,10 +33,10 @@ data class RegexFlags( val unicodeCase: Boolean = false, // u, this flags modifies behaviour of "i" flag val dotAll: Boolean = false, // s val unixLines: Boolean = false, // d + val unicodeCharacterClass: Boolean = false, // U // recognised but not yet implemented, validate() throws on these val multiline: Boolean = false, // m - val unicodeCharacterClass: Boolean = false, // U val comments: Boolean = false, // x ) { @@ -117,9 +117,8 @@ data class RegexFlags( * Call this after merging, before recursing into the flagged disjunction. */ fun validate() { - if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported") - if (unicodeCharacterClass) throw IllegalStateException("Regex flag 'U' (UNICODE_CHARACTER_CLASS) is not yet supported") - if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported") + if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported") + if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported") } /** diff --git a/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt b/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt index 8a941c9030..1ea5e0265a 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt @@ -182,6 +182,53 @@ class UnicodeCache { "javaUpperCase" to { cp -> Character.isUpperCase(cp) }, ) + // POSIX ESCAPES IN UNICODE CHAR CLASS MODE, case-insensitive, no prefix + private val unicodeCharClassModePredicates: Map Boolean> = mapOf( + "w" to { cp -> getPredicate("Isalphabetic")(cp) + || getPredicate("Isdigit")(cp) + || getPredicate("Isjoincontrol")(cp) + || getPredicate("gc=M")(cp) + }, + "d" to getPredicate("Isdigit"), + "s" to getPredicate("Iswhitespace"), + + + "lower" to getPredicate("Islowercase"), + "upper" to getPredicate("Isuppercase"), + "ascii" to { cp -> cp in 0..0x7f }, + "alpha" to getPredicate("Isalphabetic"), + "digit" to getPredicate("Isdigit"), + + "alnum" to { cp -> getPredicate("Isalphabetic")(cp) + || getPredicate("Isdigit")(cp) }, + + "punct" to getPredicate("Ispunctuation"), + + "graph" to { cp -> !(getPredicate("Iswhitespace")(cp) + || getPredicate("gc=Cc")(cp) + || getPredicate("gc=Cs")(cp) + || getPredicate("gc=Cn")(cp)) + }, + + "blank" to { cp -> getPredicate("Iswhitespace")(cp) + && !( getPredicate("gc=Zl")(cp) + || getPredicate("gc=Zp")(cp) + || cp in 0x0a..0x0d || cp == 0x85 ) + }, + + "cntrl" to getPredicate("gc=Cc"), + + "print" to { cp -> (getPredicate("graph")(cp) + || getPredicate("blank")(cp)) + && !getPredicate("cntrl")(cp) + }, + + "xdigit" to { cp -> getPredicate("gc=Nd")(cp) + || getPredicate("Ishexdigit")(cp) }, + + "space" to getPredicate("Iswhitespace"), + ) + /* We need to normalize keys so that "gc=L" and "L" point to the same thing to avoid unnecessary re-computation. Keywords are always case-insensitive, prefixes are always case-sensitive. Most properties need either a keyword @@ -237,8 +284,13 @@ class UnicodeCache { return pEscapeLabel } + // Unicode character class mode: exact match, case-sensitive + if (pEscapeLabel in unicodeCharClassModePredicates) { + return pEscapeLabel.lowercase() + } + // no match found - throw IllegalArgumentException("Unsupported/illegal category, binary property or java method") + throw IllegalArgumentException("Unsupported/illegal category, binary property or java method: $pEscapeLabel") } /* @@ -292,18 +344,11 @@ class UnicodeCache { key } - val predicate = when (key) { - in scriptPredicates -> scriptPredicates[key] - in blockPredicates -> blockPredicates[key] - in generalCategoryPredicates -> generalCategoryPredicates[key] - in binaryPropertiesPredicates -> binaryPropertiesPredicates[key] - in javaCharacterMethodPredicates -> javaCharacterMethodPredicates[key] - else -> null - } + val predicate = getPredicate(key) // first we compute and cache the base key (non-negated) cache.computeIfAbsent(key) { - computeRanges(key, predicate!!) + computeRanges(key, predicate) } // if the base kay was requested just return @@ -314,4 +359,14 @@ class UnicodeCache { MultiCharacterRange(true, cache[key]!!.ranges) } } + + private fun getPredicate(key: String): ((Int) -> Boolean) = when (key) { + in scriptPredicates -> scriptPredicates[key]!! + in blockPredicates -> blockPredicates[key]!! + in generalCategoryPredicates -> generalCategoryPredicates[key]!! + in binaryPropertiesPredicates -> binaryPropertiesPredicates[key]!! + in javaCharacterMethodPredicates -> javaCharacterMethodPredicates[key]!! + in unicodeCharClassModePredicates -> unicodeCharClassModePredicates[key]!! + else -> throw IllegalArgumentException("Unsupported/illegal category, binary property or java method") + } } \ No newline at end of file From 5617ef92e76a53927fdc8c9a44cad3f04478b2c9 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 23:57:45 -0300 Subject: [PATCH 06/12] Java regex added embedded "U" (UNICODE_CHARACTER_CLASS) flag. --- .../gene/regex/CharacterClassEscapeRxGene.kt | 36 +++++++++++-------- .../org/evomaster/core/utils/UnicodeCache.kt | 7 ++-- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt index 7525d6c9cf..590110fb74 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt @@ -65,19 +65,19 @@ class CharacterClassEscapeRxGene( private val nonVerticalSpaceMultiCharRange = MultiCharacterRange(true, verticalSpaceSet) private val posixAsciiMultiCharRange: Map = mapOf( - "lower" to listOf(CharacterRange('a', 'z')), - "upper" to listOf(CharacterRange('A', 'Z')), - "ascii" to listOf(CharacterRange(0, 0x7f)), - "alpha" to asciiLetterSet, - "digit" to digitSet, - "alnum" to digitSet + asciiLetterSet, - "punct" to punctuationSet, - "graph" to digitSet + asciiLetterSet + punctuationSet, - "print" to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"), - "blank" to stringToListOfCharacterRanges(" \t"), - "cntrl" to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"), - "xdigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')), - "space" to spaceSet, + "Lower" to listOf(CharacterRange('a', 'z')), + "Upper" to listOf(CharacterRange('A', 'Z')), + "ASCII" to listOf(CharacterRange(0, 0x7f)), + "Alpha" to asciiLetterSet, + "Digit" to digitSet, + "Alnum" to digitSet + asciiLetterSet, + "Punct" to punctuationSet, + "Graph" to digitSet + asciiLetterSet + punctuationSet, + "Print" to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"), + "Blank" to stringToListOfCharacterRanges(" \t"), + "Cntrl" to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"), + "XDigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')), + "Space" to spaceSet, ) // create both normal and negated version for all .flatMap { (key, value) -> @@ -106,7 +106,11 @@ class CharacterClassEscapeRxGene( throw IllegalArgumentException("Invalid type: $type") } - multiCharRange = if ( flags.unicodeCharacterClass && (type[0] in "wWdDsS" || (type[0] in "pP" && type.substring(2, type.length - 1).lowercase() in posixAsciiMultiCharRange)) ) { + val lowercasePosixKeys = posixAsciiMultiCharRange.keys.map{ it.lowercase() } + + multiCharRange = if ( flags.unicodeCharacterClass && + (type[0] in "wWdDsS" || + (type[0] in "pP" && type.substring(2, type.length - 1).lowercase() in lowercasePosixKeys)) ) { // UNICODE_CHARACTER_CLASSES flag is on, so these should now be in conformance with the recommendation of // Annex C: Compatibility Properties of Unicode Regular Expression, see: // https://www.unicode.org/reports/tr18/#Compatibility_Properties @@ -133,9 +137,11 @@ class CharacterClassEscapeRxGene( 'p', 'P' -> { val pLabel = type.substring(2, type.length - 1) val negated = type[0].isUpperCase() - val lookupKey = (if (negated) "^$pLabel" else pLabel).lowercase() + val lookupKey = if (negated) "^$pLabel" else pLabel if (lookupKey in posixAsciiMultiCharRange) { posixAsciiMultiCharRange[lookupKey]!! + } else if (lookupKey.lowercase() in lowercasePosixKeys) { + throw IllegalStateException("This escape (\\$type) is only valid when the \"U\" flag is on.") } else { unicodeCache.getRanges(pLabel, negated) } diff --git a/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt b/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt index 1ea5e0265a..556a36bbe2 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt @@ -284,9 +284,10 @@ class UnicodeCache { return pEscapeLabel } - // Unicode character class mode: exact match, case-sensitive - if (pEscapeLabel in unicodeCharClassModePredicates) { - return pEscapeLabel.lowercase() + // Unicode character class mode: exact match, case-insensitive + val unicodeModeKey = pEscapeLabel.lowercase() + if (unicodeModeKey in unicodeCharClassModePredicates) { + return unicodeModeKey } // no match found From 2dc9054182ae8b2ab39fe986bb396ba0f794c068 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 23:59:47 -0300 Subject: [PATCH 07/12] Added some tests. --- .../core/parser/GeneRegexJavaVisitorTest.kt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index 73edc55d60..f91253ce74 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -95,6 +95,8 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { fun testPosixCharacterClasses(){ checkSameAsJava("""\p{Lower}\p{Upper}\p{ASCII}\p{Alpha}\p{Digit}\p{Alnum}\p{Punct}\p{Graph} |\p{Print}\p{Blank}\p{Cntrl}\p{XDigit}\p{Space}""".trimMargin()) + checkSameAsJava("""(?U)\p{Lower}\p{Upper}\p{ASCII}\p{Alpha}\p{Digit}\p{Alnum}\p{Punct}\p{Graph} + |\p{pRINT}\p{BLANK}\p{cNtRl}\p{XdIgIt}\p{space}""".trimMargin()) } @Test @@ -119,7 +121,6 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("\\p{$label}") checkSameAsJava("\\P{$label}") } - checkSameAsJava("""Pe""") } @Test @@ -135,6 +136,19 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("""\P{Lower}\P{Upper}\P{ASCII}\P{Alpha}\P{Digit}\P{Alnum}\P{Punct}\P{Graph} |\P{Print}\P{Blank}\P{Cntrl}\P{XDigit}\P{Space}""".trimMargin()) checkSameAsJava("""\P{Pe}""") + checkSameAsJava("""(?U)\P{Lower}""") + checkSameAsJava("""(?U)\P{Upper}""") + checkSameAsJava("""(?U)\P{ASCII}""") + checkSameAsJava("""(?U)\P{Alpha}""") + checkSameAsJava("""(?U)\P{Digit}""") + checkSameAsJava("""(?U)\P{Alnum}""") + checkSameAsJava("""(?U)\P{Punct}""") + checkSameAsJava("""(?U)\P{Graph}""") + checkSameAsJava("""(?U)\P{Print}""") + checkSameAsJava("""(?U)\P{Blank}""") + checkSameAsJava("""(?U)\P{Cntrl}""") + checkSameAsJava("""(?U)\P{XDigit}""") + checkSameAsJava("""(?U)\P{Space}""") } @Test @@ -234,6 +248,9 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("^((?iu)@.+)$") checkSameAsJava("^(?iu)") checkSameAsJava("(?iu)") + checkSameAsJava("(?s).+") + checkSameAsJava("(?d).+") + checkSameAsJava("(?ds).+") } @Test From c14e880ed6dc4b6a86fd959bba6129d7a177492d Mon Sep 17 00:00:00 2001 From: lmasroca Date: Sun, 31 May 2026 23:59:47 -0300 Subject: [PATCH 08/12] Added some tests. --- .../core/parser/GeneRegexJavaVisitorTest.kt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index 73edc55d60..c791d78f0c 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -95,6 +95,8 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { fun testPosixCharacterClasses(){ checkSameAsJava("""\p{Lower}\p{Upper}\p{ASCII}\p{Alpha}\p{Digit}\p{Alnum}\p{Punct}\p{Graph} |\p{Print}\p{Blank}\p{Cntrl}\p{XDigit}\p{Space}""".trimMargin()) + checkSameAsJava("""(?U)\p{Lower}\p{Upper}\p{ASCII}\p{Alpha}\p{Digit}\p{Alnum}\p{Punct}\p{Graph} + |\p{pRINT}\p{BLANK}\p{cNtRl}\p{XdIgIt}\p{space}""".trimMargin()) } @Test @@ -135,6 +137,19 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("""\P{Lower}\P{Upper}\P{ASCII}\P{Alpha}\P{Digit}\P{Alnum}\P{Punct}\P{Graph} |\P{Print}\P{Blank}\P{Cntrl}\P{XDigit}\P{Space}""".trimMargin()) checkSameAsJava("""\P{Pe}""") + checkSameAsJava("""(?U)\P{Lower}""") + checkSameAsJava("""(?U)\P{Upper}""") + checkSameAsJava("""(?U)\P{ASCII}""") + checkSameAsJava("""(?U)\P{Alpha}""") + checkSameAsJava("""(?U)\P{Digit}""") + checkSameAsJava("""(?U)\P{Alnum}""") + checkSameAsJava("""(?U)\P{Punct}""") + checkSameAsJava("""(?U)\P{Graph}""") + checkSameAsJava("""(?U)\P{Print}""") + checkSameAsJava("""(?U)\P{Blank}""") + checkSameAsJava("""(?U)\P{Cntrl}""") + checkSameAsJava("""(?U)\P{XDigit}""") + checkSameAsJava("""(?U)\P{Space}""") } @Test @@ -234,6 +249,9 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("^((?iu)@.+)$") checkSameAsJava("^(?iu)") checkSameAsJava("(?iu)") + checkSameAsJava("(?s).+") + checkSameAsJava("(?d).+") + checkSameAsJava("(?ds).+") } @Test From a24b79acf823fb19efc9b6ae68610aa282ebd443 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Mon, 1 Jun 2026 00:10:15 -0300 Subject: [PATCH 09/12] Re-added a test. --- .../kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index f91253ce74..c791d78f0c 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -121,6 +121,7 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("\\p{$label}") checkSameAsJava("\\P{$label}") } + checkSameAsJava("""Pe""") } @Test From 6cc8c638f06af3afedd313d2a80e7294d13efe9c Mon Sep 17 00:00:00 2001 From: lmasroca Date: Mon, 29 Jun 2026 16:20:48 -0300 Subject: [PATCH 10/12] Temporary fix for lone surrogates in rest path issue. --- .../search/gene/regex/AnyCharacterRxGene.kt | 11 ++++++++--- .../evomaster/core/utils/MultiCharacterRange.kt | 17 +++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 2c4b9f3cc6..16916432b1 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -15,6 +15,7 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.utils.CharacterRange import org.evomaster.core.utils.MultiCharacterRange import org.evomaster.core.utils.RegexFlags import org.slf4j.Logger @@ -30,9 +31,13 @@ class AnyCharacterRxGene( /** All characters except for line terminators are recognized by "." in regex, see: * https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#COMMENTS:~:text=range%20forming%20metacharacter.-,Line%20terminators,-A%20line%20terminator */ - val defaultValidRanges = MultiCharacterRange(true,"\n\r\u0085\u2028\u2029") - val dotAllValidRanges = MultiCharacterRange(true, "") // all characters accepted - val unixLinesValidRanges = MultiCharacterRange(true, "\n") + /* + TODO in Java regex lone surrogates match ".", but this causes trouble when appearing in a rest path, + so for now we avoid surrogates altogether + */ + val dotAllValidRanges = MultiCharacterRange(true,listOf(CharacterRange('\uD800','\uDFFF'))) // all characters accepted + val defaultValidRanges = MultiCharacterRange(true,"\n\r\u0085\u2028\u2029").intersect(dotAllValidRanges) + val unixLinesValidRanges = MultiCharacterRange(true, "\n").intersect(dotAllValidRanges) } var value: Char = 'a' diff --git a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt index 2a507a04dc..77c8b5f680 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt @@ -5,10 +5,6 @@ import org.slf4j.LoggerFactory class MultiCharacterRange internal constructor(val ranges: List) { - init { - require(ranges.isNotEmpty()) { "MultiCharacterRange cannot be created with an empty list" } - } - companion object { private val log = LoggerFactory.getLogger(MultiCharacterRange::class.java) @@ -31,7 +27,7 @@ class MultiCharacterRange internal constructor(val ranges: List) if (negated) { internalRanges.add(CharacterRange(Character.MIN_VALUE, Character.MAX_VALUE)) } - for (range in ranges) { + for (range in ranges.sortedBy { it.start }) { internalRanges = if (negated) { remove(internalRanges, CharacterRange(range.start, range.end)) } else { @@ -55,7 +51,7 @@ class MultiCharacterRange internal constructor(val ranges: List) var currentEnd = toAdd.end var merged = false - for ((start, end) in internalRanges.sortedBy { it.start }) { + for ((start, end) in internalRanges) { when { end.code < currentStart.code - 1 -> newInternalRanges += CharacterRange(start, end) start.code > currentEnd.code + 1 -> { @@ -201,10 +197,15 @@ class MultiCharacterRange internal constructor(val ranges: List) } currentRangeMinValue = currentRangeMaxValue } - assert(false) // internal ranges being empty should never happen - return '0' + throw IllegalStateException("Cannot sample characters from an empty char range") + } + + fun intersect(other: MultiCharacterRange): MultiCharacterRange { + return intersect(this, other) } + val isEmpty: Boolean get() = ranges.isEmpty() + val isNotEmpty: Boolean get() = ranges.isNotEmpty() val size: Int get() = ranges.size val charCount :Int = ranges.sumOf{ it.size } operator fun get(index: Int): CharacterRange = ranges[index] From 0a519712ca98dd9ec0a87f7c5d41974b94f3a366 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Mon, 29 Jun 2026 17:20:22 -0300 Subject: [PATCH 11/12] Small fix, "U" flag also implies "u" flag. --- core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt | 3 ++- .../org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt index 4dc7cbfa70..ede7789a26 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt @@ -144,7 +144,8 @@ data class RegexFlags( * and unicodeCase flag values. */ fun isCaseable(codePoint: Int): Boolean { - return if (caseInsensitive && unicodeCase) { + // unicodeCharacterClass implies also unicodeCase + return if (caseInsensitive && (unicodeCase || unicodeCharacterClass)) { Character.toUpperCase(codePoint) != Character.toLowerCase(codePoint) } else if (caseInsensitive) { diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index e1df9cd7a8..0a8e7e6928 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -417,4 +417,9 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { assertThrows { checkSameAsJava("a([b&&c])d") } assertThrows { checkSameAsJava("abc|\\k") } } + + @Test + fun testUnicodeCharClassFlagImpliesUnicodeCase(){ + checkCanSample("(?iU)Å", "å", 100) + } } \ No newline at end of file From c8804cede75e8066341e538dd6ad08cc5a944d9a Mon Sep 17 00:00:00 2001 From: lmasroca Date: Wed, 1 Jul 2026 17:50:34 -0300 Subject: [PATCH 12/12] Refactor: requested changes. --- .../search/gene/regex/AnyCharacterRxGene.kt | 17 ++++++++++------- .../gene/regex/CharacterClassEscapeRxGene.kt | 11 ++++++++--- .../org/evomaster/core/utils/RegexFlags.kt | 3 --- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 16916432b1..efc81b2117 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -21,6 +21,11 @@ import org.evomaster.core.utils.RegexFlags import org.slf4j.Logger import org.slf4j.LoggerFactory +private const val firstSurrogateChar = '\uD800' +private const val lastSurrogateChar = '\uDFFF' +private const val defaultLineTerminators = "\n\r\u0085\u2028\u2029" +private const val unixLinesModeLineTerminators = "\n" + class AnyCharacterRxGene( val flags: RegexFlags = RegexFlags() ) : RxAtom, SimpleGene(".") { @@ -35,9 +40,9 @@ class AnyCharacterRxGene( TODO in Java regex lone surrogates match ".", but this causes trouble when appearing in a rest path, so for now we avoid surrogates altogether */ - val dotAllValidRanges = MultiCharacterRange(true,listOf(CharacterRange('\uD800','\uDFFF'))) // all characters accepted - val defaultValidRanges = MultiCharacterRange(true,"\n\r\u0085\u2028\u2029").intersect(dotAllValidRanges) - val unixLinesValidRanges = MultiCharacterRange(true, "\n").intersect(dotAllValidRanges) + val dotAllValidRanges = MultiCharacterRange(true,listOf(CharacterRange(firstSurrogateChar,lastSurrogateChar))) // all characters accepted + val defaultValidRanges = MultiCharacterRange(true,defaultLineTerminators).intersect(dotAllValidRanges) + val unixLinesValidRanges = MultiCharacterRange(true, unixLinesModeLineTerminators).intersect(dotAllValidRanges) } var value: Char = 'a' @@ -69,10 +74,8 @@ class AnyCharacterRxGene( override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) { val previous = value - value = validRanges.sample(randomness) - - if (previous == value) { - randomize(randomness, tryToForceNewValue) + while(tryToForceNewValue && previous == value) { + value = validRanges.sample(randomness) } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt index 6725ec6613..3275b82bf0 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt @@ -17,6 +17,11 @@ import org.evomaster.core.utils.UnicodeCache import org.slf4j.LoggerFactory import kotlin.collections.contains +private const val firstASCIIChar = 0 +private const val lastASCIIChar = 0x7f +private val unicodeCharClassModePredefinedEscapes = setOf('w', 'W', 'd', 'D', 's', 'S') +private val posixEscapePrefix = setOf('p', 'P') + /* \w Find a word character \W Find a non-word character @@ -67,7 +72,7 @@ class CharacterClassEscapeRxGene( private val posixAsciiMultiCharRange: Map = mapOf( "Lower" to listOf(CharacterRange('a', 'z')), "Upper" to listOf(CharacterRange('A', 'Z')), - "ASCII" to listOf(CharacterRange(0, 0x7f)), + "ASCII" to listOf(CharacterRange(firstASCIIChar, lastASCIIChar)), "Alpha" to asciiLetterSet, "Digit" to digitSet, "Alnum" to digitSet + asciiLetterSet, @@ -109,8 +114,8 @@ class CharacterClassEscapeRxGene( val lowercasePosixKeys = posixAsciiMultiCharRange.keys.map{ it.lowercase() } multiCharRange = if ( flags.unicodeCharacterClass && - (type[0] in "wWdDsS" || - (type[0] in "pP" && type.substring(2, type.length - 1).lowercase() in lowercasePosixKeys)) ) { + (type[0] in unicodeCharClassModePredefinedEscapes || + (type[0] in posixEscapePrefix && type.substring(2, type.length - 1).lowercase() in lowercasePosixKeys)) ) { // UNICODE_CHARACTER_CLASSES flag is on, so these should now be in conformance with the recommendation of // Annex C: Compatibility Properties of Unicode Regular Expression, see: // https://www.unicode.org/reports/tr18/#Compatibility_Properties diff --git a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt index ede7789a26..8a923caeb6 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt @@ -30,14 +30,11 @@ data class ParsedFlagExpression( private val validFlagCharacters = setOf('i', 'u', 's', 'm', 'd', 'U', 'x') data class RegexFlags( - // currently implemented val caseInsensitive: Boolean = false, // i val unicodeCase: Boolean = false, // u, this flags modifies behaviour of "i" flag val dotAll: Boolean = false, // s val unixLines: Boolean = false, // d val unicodeCharacterClass: Boolean = false, // U - - // recognised but not yet implemented, validate() throws on these val multiline: Boolean = false, // m val comments: Boolean = false, // x ) {