diff --git a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt index 7dda4fd896..e21b12060f 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt @@ -442,7 +442,7 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege } if(ctx.DOT() != null){ - return VisitResult(AnyCharacterRxGene()) + return VisitResult(AnyCharacterRxGene(currentFlags)) } if(ctx.characterClass() != null){ diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 9a4aecb545..efc81b2117 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -15,24 +15,50 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.utils.CharacterRange +import org.evomaster.core.utils.MultiCharacterRange +import org.evomaster.core.utils.RegexFlags import org.slf4j.Logger import org.slf4j.LoggerFactory +private const val firstSurrogateChar = '\uD800' +private const val lastSurrogateChar = '\uDFFF' +private const val defaultLineTerminators = "\n\r\u0085\u2028\u2029" +private const val unixLinesModeLineTerminators = "\n" -class AnyCharacterRxGene : RxAtom, SimpleGene("."){ +class AnyCharacterRxGene( + val flags: RegexFlags = RegexFlags() +) : RxAtom, SimpleGene(".") { companion object{ private val log : Logger = LoggerFactory.getLogger(AnyCharacterRxGene::class.java) + + /** All characters except for line terminators are recognized by "." in regex, see: + * https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#COMMENTS:~:text=range%20forming%20metacharacter.-,Line%20terminators,-A%20line%20terminator + */ + /* + TODO in Java regex lone surrogates match ".", but this causes trouble when appearing in a rest path, + so for now we avoid surrogates altogether + */ + val dotAllValidRanges = MultiCharacterRange(true,listOf(CharacterRange(firstSurrogateChar,lastSurrogateChar))) // all characters accepted + val defaultValidRanges = MultiCharacterRange(true,defaultLineTerminators).intersect(dotAllValidRanges) + val unixLinesValidRanges = MultiCharacterRange(true, unixLinesModeLineTerminators).intersect(dotAllValidRanges) } var value: Char = 'a' + val validRanges = when { + flags.dotAll -> dotAllValidRanges + flags.unixLines -> unixLinesValidRanges + else -> defaultValidRanges + } + override fun checkForLocallyValidIgnoringChildren() : Boolean{ return true } override fun copyContent(): Gene { - val copy = AnyCharacterRxGene() + val copy = AnyCharacterRxGene(flags) copy.value = this.value copy.name = this.name //in case name is changed from its default return copy @@ -46,8 +72,11 @@ class AnyCharacterRxGene : RxAtom, SimpleGene("."){ } override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) { - //TODO properly... this is just a tmp hack - value = randomness.nextWordChar() + val previous = value + + while(tryToForceNewValue && previous == value) { + value = validRanges.sample(randomness) + } } override fun shallowMutate(randomness: Randomness, apc: AdaptiveParameterControl, mwc: MutationWeightControl, selectionStrategy: SubsetGeneMutationSelectionStrategy, enableAdaptiveGeneMutation: Boolean, additionalGeneMutationInfo: AdditionalGeneMutationInfo?): Boolean { diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt index 5abc86b9ba..3275b82bf0 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt @@ -17,6 +17,11 @@ import org.evomaster.core.utils.UnicodeCache import org.slf4j.LoggerFactory import kotlin.collections.contains +private const val firstASCIIChar = 0 +private const val lastASCIIChar = 0x7f +private val unicodeCharClassModePredefinedEscapes = setOf('w', 'W', 'd', 'D', 's', 'S') +private val posixEscapePrefix = setOf('p', 'P') + /* \w Find a word character \W Find a non-word character @@ -65,20 +70,20 @@ class CharacterClassEscapeRxGene( private val nonVerticalSpaceMultiCharRange = MultiCharacterRange(true, verticalSpaceSet) private val posixAsciiMultiCharRange: Map = mapOf( - "Lower" to listOf(CharacterRange('a', 'z')), - "Upper" to listOf(CharacterRange('A', 'Z')), - "ASCII" to listOf(CharacterRange(0, 0x7f)), - "Alpha" to asciiLetterSet, - "Digit" to digitSet, - "Alnum" to digitSet + asciiLetterSet, - "Punct" to punctuationSet, - "Graph" to digitSet + asciiLetterSet + punctuationSet, - "Print" to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"), - "Blank" to stringToListOfCharacterRanges(" \t"), - "Cntrl" to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"), - "XDigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')), - "Space" to spaceSet, - ) + "Lower" to listOf(CharacterRange('a', 'z')), + "Upper" to listOf(CharacterRange('A', 'Z')), + "ASCII" to listOf(CharacterRange(firstASCIIChar, lastASCIIChar)), + "Alpha" to asciiLetterSet, + "Digit" to digitSet, + "Alnum" to digitSet + asciiLetterSet, + "Punct" to punctuationSet, + "Graph" to digitSet + asciiLetterSet + punctuationSet, + "Print" to digitSet + asciiLetterSet + punctuationSet + stringToListOfCharacterRanges("\u0020"), + "Blank" to stringToListOfCharacterRanges(" \t"), + "Cntrl" to listOf(CharacterRange(0, 0x1f)) + stringToListOfCharacterRanges("\u007f"), + "XDigit" to listOf(CharacterRange('0', '9'), CharacterRange('a', 'f'), CharacterRange('A', 'F')), + "Space" to spaceSet, + ) // create both normal and negated version for all .flatMap { (key, value) -> listOf( @@ -106,29 +111,49 @@ class CharacterClassEscapeRxGene( throw IllegalArgumentException("Invalid type: $type") } - multiCharRange = when(type[0]){ - 'w' -> wordMultiCharRange - 'W' -> nonWordMultiCharRange - 'd' -> digitMultiCharRange - 'D' -> nonDigitMultiCharRange - 's' -> spaceMultiCharRange - 'S' -> nonSpaceMultiCharRange - 'v' -> verticalSpaceMultiCharRange - 'V' -> nonVerticalSpaceMultiCharRange - 'h' -> horizontalSpaceMultiCharRange - 'H' -> nonHorizontalSpaceMultiCharRange - 'p', 'P' -> { - val pLabel = type.substring(2, type.length - 1) - val negated = type[0].isUpperCase() - val lookupKey = if (negated) "^$pLabel" else pLabel - if (lookupKey !in posixAsciiMultiCharRange) { - unicodeCache.getRanges(pLabel, negated) - } else { - posixAsciiMultiCharRange[lookupKey]!! + val lowercasePosixKeys = posixAsciiMultiCharRange.keys.map{ it.lowercase() } + + multiCharRange = if ( flags.unicodeCharacterClass && + (type[0] in unicodeCharClassModePredefinedEscapes || + (type[0] in posixEscapePrefix && type.substring(2, type.length - 1).lowercase() in lowercasePosixKeys)) ) { + // UNICODE_CHARACTER_CLASSES flag is on, so these should now be in conformance with the recommendation of + // Annex C: Compatibility Properties of Unicode Regular Expression, see: + // https://www.unicode.org/reports/tr18/#Compatibility_Properties + val cacheLabel = when (type[0]) { + 'd', 'D', 's', 'S', 'w', 'W' -> type.lowercase() + 'p', 'P' -> type.substring(2, type.length - 1) + else -> //this should never happen due to check in init + throw IllegalStateException("Type '\\$type' not supported yet") + } + unicodeCache.getRanges(cacheLabel, type[0].isUpperCase()) + } else { + // regular predefined character classes + when(type[0]){ + 'w' -> wordMultiCharRange + 'W' -> nonWordMultiCharRange + 'd' -> digitMultiCharRange + 'D' -> nonDigitMultiCharRange + 's' -> spaceMultiCharRange + 'S' -> nonSpaceMultiCharRange + 'v' -> verticalSpaceMultiCharRange + 'V' -> nonVerticalSpaceMultiCharRange + 'h' -> horizontalSpaceMultiCharRange + 'H' -> nonHorizontalSpaceMultiCharRange + 'p', 'P' -> { + val pLabel = type.substring(2, type.length - 1) + val negated = type[0].isUpperCase() + val lookupKey = if (negated) "^$pLabel" else pLabel + if (lookupKey in posixAsciiMultiCharRange) { + posixAsciiMultiCharRange[lookupKey]!! + } else if (lookupKey.lowercase() in lowercasePosixKeys) { + throw IllegalStateException("This escape (\\$type) is only valid when the \"U\" flag is on.") + } else { + unicodeCache.getRanges(pLabel, negated) + } } + else -> //this should never happen due to check in init + throw IllegalStateException("Type '\\$type' not supported yet") } - else -> //this should never happen due to check in init - throw IllegalStateException("Type '\\$type' not supported yet") } } diff --git a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt index 3056323358..77c8b5f680 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt @@ -200,6 +200,10 @@ class MultiCharacterRange internal constructor(val ranges: List) throw IllegalStateException("Cannot sample characters from an empty char range") } + fun intersect(other: MultiCharacterRange): MultiCharacterRange { + return intersect(this, other) + } + val isEmpty: Boolean get() = ranges.isEmpty() val isNotEmpty: Boolean get() = ranges.isNotEmpty() val size: Int get() = ranges.size diff --git a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt index 6f977cc781..8a923caeb6 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt @@ -30,15 +30,12 @@ data class ParsedFlagExpression( private val validFlagCharacters = setOf('i', 'u', 's', 'm', 'd', 'U', 'x') data class RegexFlags( - // currently implemented val caseInsensitive: Boolean = false, // i val unicodeCase: Boolean = false, // u, this flags modifies behaviour of "i" flag - - // recognised but not yet implemented, validate() throws on these val dotAll: Boolean = false, // s - val multiline: Boolean = false, // m val unixLines: Boolean = false, // d val unicodeCharacterClass: Boolean = false, // U + val multiline: Boolean = false, // m val comments: Boolean = false, // x ) { @@ -135,10 +132,7 @@ data class RegexFlags( * Call this after merging, before recursing into the flagged disjunction. */ fun validate() { - if (dotAll) throw IllegalStateException("Regex flag 's' (DOTALL) is not yet supported") - if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported") - if (unixLines) throw IllegalStateException("Regex flag 'd' (UNIX_LINES) is not yet supported") - if (unicodeCharacterClass) throw IllegalStateException("Regex flag 'U' (UNICODE_CHARACTER_CLASS) is not yet supported") + if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported") if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported") } @@ -147,7 +141,8 @@ data class RegexFlags( * and unicodeCase flag values. */ fun isCaseable(codePoint: Int): Boolean { - return if (caseInsensitive && unicodeCase) { + // unicodeCharacterClass implies also unicodeCase + return if (caseInsensitive && (unicodeCase || unicodeCharacterClass)) { Character.toUpperCase(codePoint) != Character.toLowerCase(codePoint) } else if (caseInsensitive) { diff --git a/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt b/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt index 8a941c9030..556a36bbe2 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/UnicodeCache.kt @@ -182,6 +182,53 @@ class UnicodeCache { "javaUpperCase" to { cp -> Character.isUpperCase(cp) }, ) + // POSIX ESCAPES IN UNICODE CHAR CLASS MODE, case-insensitive, no prefix + private val unicodeCharClassModePredicates: Map Boolean> = mapOf( + "w" to { cp -> getPredicate("Isalphabetic")(cp) + || getPredicate("Isdigit")(cp) + || getPredicate("Isjoincontrol")(cp) + || getPredicate("gc=M")(cp) + }, + "d" to getPredicate("Isdigit"), + "s" to getPredicate("Iswhitespace"), + + + "lower" to getPredicate("Islowercase"), + "upper" to getPredicate("Isuppercase"), + "ascii" to { cp -> cp in 0..0x7f }, + "alpha" to getPredicate("Isalphabetic"), + "digit" to getPredicate("Isdigit"), + + "alnum" to { cp -> getPredicate("Isalphabetic")(cp) + || getPredicate("Isdigit")(cp) }, + + "punct" to getPredicate("Ispunctuation"), + + "graph" to { cp -> !(getPredicate("Iswhitespace")(cp) + || getPredicate("gc=Cc")(cp) + || getPredicate("gc=Cs")(cp) + || getPredicate("gc=Cn")(cp)) + }, + + "blank" to { cp -> getPredicate("Iswhitespace")(cp) + && !( getPredicate("gc=Zl")(cp) + || getPredicate("gc=Zp")(cp) + || cp in 0x0a..0x0d || cp == 0x85 ) + }, + + "cntrl" to getPredicate("gc=Cc"), + + "print" to { cp -> (getPredicate("graph")(cp) + || getPredicate("blank")(cp)) + && !getPredicate("cntrl")(cp) + }, + + "xdigit" to { cp -> getPredicate("gc=Nd")(cp) + || getPredicate("Ishexdigit")(cp) }, + + "space" to getPredicate("Iswhitespace"), + ) + /* We need to normalize keys so that "gc=L" and "L" point to the same thing to avoid unnecessary re-computation. Keywords are always case-insensitive, prefixes are always case-sensitive. Most properties need either a keyword @@ -237,8 +284,14 @@ class UnicodeCache { return pEscapeLabel } + // Unicode character class mode: exact match, case-insensitive + val unicodeModeKey = pEscapeLabel.lowercase() + if (unicodeModeKey in unicodeCharClassModePredicates) { + return unicodeModeKey + } + // no match found - throw IllegalArgumentException("Unsupported/illegal category, binary property or java method") + throw IllegalArgumentException("Unsupported/illegal category, binary property or java method: $pEscapeLabel") } /* @@ -292,18 +345,11 @@ class UnicodeCache { key } - val predicate = when (key) { - in scriptPredicates -> scriptPredicates[key] - in blockPredicates -> blockPredicates[key] - in generalCategoryPredicates -> generalCategoryPredicates[key] - in binaryPropertiesPredicates -> binaryPropertiesPredicates[key] - in javaCharacterMethodPredicates -> javaCharacterMethodPredicates[key] - else -> null - } + val predicate = getPredicate(key) // first we compute and cache the base key (non-negated) cache.computeIfAbsent(key) { - computeRanges(key, predicate!!) + computeRanges(key, predicate) } // if the base kay was requested just return @@ -314,4 +360,14 @@ class UnicodeCache { MultiCharacterRange(true, cache[key]!!.ranges) } } + + private fun getPredicate(key: String): ((Int) -> Boolean) = when (key) { + in scriptPredicates -> scriptPredicates[key]!! + in blockPredicates -> blockPredicates[key]!! + in generalCategoryPredicates -> generalCategoryPredicates[key]!! + in binaryPropertiesPredicates -> binaryPropertiesPredicates[key]!! + in javaCharacterMethodPredicates -> javaCharacterMethodPredicates[key]!! + in unicodeCharClassModePredicates -> unicodeCharClassModePredicates[key]!! + else -> throw IllegalArgumentException("Unsupported/illegal category, binary property or java method") + } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index e7ea3ed26d..0a8e7e6928 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -96,6 +96,8 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { fun testPosixCharacterClasses(){ checkSameAsJava("""\p{Lower}\p{Upper}\p{ASCII}\p{Alpha}\p{Digit}\p{Alnum}\p{Punct}\p{Graph} |\p{Print}\p{Blank}\p{Cntrl}\p{XDigit}\p{Space}""".trimMargin()) + checkSameAsJava("""(?U)\p{Lower}\p{Upper}\p{ASCII}\p{Alpha}\p{Digit}\p{Alnum}\p{Punct}\p{Graph} + |\p{pRINT}\p{BLANK}\p{cNtRl}\p{XdIgIt}\p{space}""".trimMargin()) } @Test @@ -136,6 +138,19 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("""\P{Lower}\P{Upper}\P{ASCII}\P{Alpha}\P{Digit}\P{Alnum}\P{Punct}\P{Graph} |\P{Print}\P{Blank}\P{Cntrl}\P{XDigit}\P{Space}""".trimMargin()) checkSameAsJava("""\P{Pe}""") + checkSameAsJava("""(?U)\P{Lower}""") + checkSameAsJava("""(?U)\P{Upper}""") + checkSameAsJava("""(?U)\P{ASCII}""") + checkSameAsJava("""(?U)\P{Alpha}""") + checkSameAsJava("""(?U)\P{Digit}""") + checkSameAsJava("""(?U)\P{Alnum}""") + checkSameAsJava("""(?U)\P{Punct}""") + checkSameAsJava("""(?U)\P{Graph}""") + checkSameAsJava("""(?U)\P{Print}""") + checkSameAsJava("""(?U)\P{Blank}""") + checkSameAsJava("""(?U)\P{Cntrl}""") + checkSameAsJava("""(?U)\P{XDigit}""") + checkSameAsJava("""(?U)\P{Space}""") } @Test @@ -235,6 +250,9 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("^((?iu)@.+)$") checkSameAsJava("^(?iu)") checkSameAsJava("(?iu)") + checkSameAsJava("(?s).+") + checkSameAsJava("(?d).+") + checkSameAsJava("(?ds).+") } @Test @@ -399,4 +417,9 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { assertThrows { checkSameAsJava("a([b&&c])d") } assertThrows { checkSameAsJava("abc|\\k") } } + + @Test + fun testUnicodeCharClassFlagImpliesUnicodeCase(){ + checkCanSample("(?iU)Å", "å", 100) + } } \ No newline at end of file