perf: optimize re filter

frac/fraction_test.go

@@ -352,7 +352,7 @@ func (s *FractionTestSuite) TestSearchRe() {
 	s.AssertSearch(`v:re("^\[(ERROR|FATAL)\]$")`, docs, []int{6})
 	// In tests we transform keyword token to lower-case.
 	// So case-sensitive expression will always yield nothing.
-	s.AssertSearch(`v:re("(?-i)^\[(ERROR|FATAL)\]$")`, docs, []int{})
+	s.AssertSearch(`v:re("(?-i)^\[(ERROR|FATAL)\]$")`, docs, []int{6})
 }
 
 func (s *FractionTestSuite) TestSearchIPRange() {

parser/seqql_filter_test.go

@@ -395,8 +395,8 @@ func TestParseSeqQLError(t *testing.T) {
 	test(`keyword:re()`, "parsing `re` filter: invalid syntax")
 	test(`keyword:re(")`, "parsing `re` filter: invalid syntax")
 	test(`keyword:re(""invalid)`, "parsing `re` filter: expected ')', got \"invalid\"")
-	test(`keyword:re("[invalid")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: missing closing ]: `[invalid)$`")
-	test(`keyword:re("invalid)")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: unexpected ): `^(?i:invalid))$`")
+	test(`keyword:re("[invalid")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: missing closing ]: `[invalid`")
+	test(`keyword:re("invalid)")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: unexpected ): `invalid)`")
 	test(`keyword:re("[z-a]")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: invalid character class range: `z-a`")
 	test(`keyword:re("*invalid")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: missing argument to repetition operator: `*`")
 

parser/token_re.go

@@ -3,13 +3,28 @@ package parser
 import (
 	"fmt"
 	"regexp"
+	"regexp/syntax"
+	"slices"
 	"strings"
+
+	"github.com/ozontech/seq-db/config"
+	"github.com/ozontech/seq-db/util"
 )
 
+type ReLiteral struct {
+	Value    []byte
+	Foldable bool
+}
+
 type Re struct {
-	Field              string
+	Field string
+
 	Expression         Term
 	CompiledExpression *regexp.Regexp
+
+	Prefix ReLiteral
+	Middle []ReLiteral
+	Suffix ReLiteral
 }
 
 func (r *Re) DumpSeqQL(b *strings.Builder) {
@@ -43,6 +58,12 @@ func parseReFilter(lex *lexer, fieldName string) (*Re, error) {
 	// This behaviour has negative impact on language extendability.
 	expr := lex.Token
 
+	lex.Next()
+	if !lex.IsKeyword(")") {
+		return nil, fmt.Errorf("expected ')', got %q", lex.Token)
+	}
+	lex.Next()
+
 	// Here are two important things to keep in mind:
 	//  - We perform case-insensitive search by default;
 	//  - We force anchoring for the expression;
@@ -52,22 +73,161 @@ func parseReFilter(lex *lexer, fieldName string) (*Re, error) {
 	//
 	// See Prometheus TSDB `FastRegexMatcher` for a similar approach:
 	// https://github.com/prometheus/prometheus/blob/19fd0b0b1dbfe01a5e49f5d04544a7c5853c12bb/model/labels/regexp.go#L70
-	expr = "^(?i:" + expr + ")$"
 
-	compiled, err := regexp.Compile(expr)
+	re, err := syntax.Parse(expr, syntax.Perl)
 	if err != nil {
 		return nil, fmt.Errorf("invalid expression for `re` filter: %s", err)
 	}
 
-	lex.Next()
-	if !lex.IsKeyword(")") {
-		return nil, fmt.Errorf("expected ')', got %q", lex.Token)
+	// NOTE(dkharms): We do not allow overriding of case-sensitivity
+	// in case if search is case-insensitive.
+	//
+	// This way `re` filter works consistently with how `keyword`
+	// and `text` search behave.
+	overridable := config.CaseSensitive
+	if !overridable && hasCaseSensitivityOverride(re) {
+		return nil, fmt.Errorf(
+			"store is configured for case-insensitive search: " +
+				"you cannot override this option",
+		)
 	}
 
-	lex.Next()
+	// NOTE(dkharms): Again, if store works in case-insensitive mode
+	// we simulate behaviour of `keyword` and `text` indexes.
+	// Should we really do this?
+	if !overridable {
+		expr = strings.ToLower(expr)
+	}
+
+	// NOTE(dkharms): We force anchoring for the expression.
+	// Anchoring is necessary for prefix and suffix search optimization.
+	expr = "^" + expr + "$"
+	exp, err := regexp.Compile(expr)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"it's likely you've encountered a bug: " +
+				"please contact seq-db team",
+		)
+	}
+
+	re = optimizeRe(re)
+
 	return &Re{
-		Field:              fieldName,
+		Field: fieldName,
+
 		Expression:         newTextTerm(expr),
-		CompiledExpression: compiled,
+		CompiledExpression: exp,
+
+		Prefix: prefix(re),
+		Middle: middle(re),
+		Suffix: suffix(re),
 	}, nil
 }
+
+func hasCaseSensitivityOverride(re *syntax.Regexp) bool {
+	switch re.Op {
+	case syntax.OpLiteral, syntax.OpCharClass:
+		return (re.Flags & syntax.FoldCase) == syntax.FoldCase
+	default:
+		return slices.ContainsFunc(re.Sub, hasCaseSensitivityOverride)
+	}
+}
+
+// eliminateCapture transforms regular expression into
+// semantically equivalent one but without capturing groups.
+func eliminateCapture(re *syntax.Regexp) {
+	if re.Op == syntax.OpCapture {
+		*re = *re.Sub[0]
+	}
+
+	for _, s := range re.Sub {
+		eliminateCapture(s)
+	}
+}
+
+func optimizeRe(re *syntax.Regexp) *syntax.Regexp {
+	// Eliminate captures in-place.
+	//
+	// They do not change semantics of regular expression
+	// but make simplification and extraction of literals more difficult.
+	eliminateCapture(re)
+
+	// Well, some simplification are not reflected at already parsed tree.
+	// So we do it again :)
+	re, err := syntax.Parse(re.Simplify().String(), syntax.Perl)
+	if err != nil {
+		panic(fmt.Sprintf("BUG: cannot parse re after optimization pass: %s", err))
+	}
+
+	return re
+}
+
+func prefix(re *syntax.Regexp) ReLiteral {
+	// For example, we work with regular expression `seqdb-(stg|prod)-[1-9]+`.
+	subs := []*syntax.Regexp{re}
+
+	// Yep, this is concatention. We are interesed in its subexpressions.
+	if subs[0].Op == syntax.OpConcat {
+		subs = subs[0].Sub
+	}
+
+	// Skip symbols like `^`.
+	if subs[0].Op == syntax.OpBeginText || subs[0].Op == syntax.OpBeginLine {
+		subs = subs[1:]
+	}
+
+	// Well, not today.
+	if len(subs) == 0 || subs[0].Op != syntax.OpLiteral {
+		return ReLiteral{}
+	}
+
+	return ReLiteral{
+		// TODO(dkharms): Check whether it is safe.
+		Value:    util.StringToByteUnsafe(string(subs[0].Rune)),
+		Foldable: (subs[0].Flags & syntax.FoldCase) == syntax.FoldCase,
+	}
+}
+
+func middle(re *syntax.Regexp) []ReLiteral {
+	var m []ReLiteral
+
+	subs := []*syntax.Regexp{re}
+	if subs[0].Op == syntax.OpConcat {
+		subs = subs[0].Sub
+	}
+
+	for len(subs) > 0 {
+		if subs[0].Op == syntax.OpLiteral {
+			m = append(m, ReLiteral{
+				// TODO(dkharms): Check whether it is safe.
+				Value:    util.StringToByteUnsafe(string(subs[0].Rune)),
+				Foldable: (subs[0].Flags & syntax.FoldCase) == syntax.FoldCase,
+			})
+		}
+		subs = subs[1:]
+	}
+
+	return m
+}
+
+func suffix(re *syntax.Regexp) ReLiteral {
+	subs := []*syntax.Regexp{re}
+
+	if subs[0].Op == syntax.OpConcat {
+		subs = subs[0].Sub
+	}
+
+	if subs[len(subs)-1].Op == syntax.OpBeginText || subs[len(subs)-1].Op == syntax.OpBeginLine {
+		subs = subs[:len(subs)-1]
+	}
+
+	if len(subs) == 0 || subs[len(subs)-1].Op != syntax.OpLiteral {
+		return ReLiteral{}
+	}
+
+	return ReLiteral{
+		// TODO(dkharms): Check whether it is safe.
+		Value:    util.StringToByteUnsafe(string(subs[len(subs)-1].Rune)),
+		Foldable: (subs[len(subs)-1].Flags & syntax.FoldCase) == syntax.FoldCase,
+	}
+}

parser/token_re_test.go

@@ -0,0 +1,144 @@
+package parser
+
+import (
+	"regexp/syntax"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptimizationPass(t *testing.T) {
+	parse := func(s string) *syntax.Regexp {
+		t.Helper()
+
+		re, err := syntax.Parse(s, syntax.Perl)
+		require.NoError(t, err)
+
+		return re
+	}
+
+	cases := []struct {
+		re       string
+		expected string
+	}{
+		{
+			// Remove capturing groups.
+			re:       "(a(b(c(d))))",
+			expected: "abcd",
+		},
+		{
+			re: "(seqdb-prod)?",
+			// This is non-capturing group.
+			expected: "(?:seqdb-prod)?",
+		},
+		{
+			re: "[1][2][3]",
+			// This is non-capturing group.
+			expected: "123",
+		},
+	}
+
+	for _, cc := range cases {
+		assert.Equal(t, cc.expected, optimizeRe(parse(cc.re)).String())
+	}
+}
+
+func TestLiteralExtraction(t *testing.T) {
+	parse := func(s string) *syntax.Regexp {
+		t.Helper()
+
+		re, err := syntax.Parse(s, syntax.Perl)
+		require.NoError(t, err)
+
+		return optimizeRe(re)
+	}
+
+	prefix := func(re *syntax.Regexp) []ReLiteral {
+		if r := prefix(re); len(r.Value) > 0 {
+			return []ReLiteral{r}
+		}
+		return []ReLiteral{}
+	}
+
+	suffix := func(re *syntax.Regexp) []ReLiteral {
+		if r := suffix(re); len(r.Value) > 0 {
+			return []ReLiteral{r}
+		}
+		return []ReLiteral{}
+	}
+
+	cases := []struct {
+		re       *syntax.Regexp
+		expected []ReLiteral
+		fn       func(*syntax.Regexp) []ReLiteral
+	}{
+		{
+			re:       parse("simple"),
+			expected: []ReLiteral{{Value: []byte("simple"), Foldable: false}},
+			fn:       prefix,
+		},
+		{
+			re:       parse("easy-prefix-[a-zA-Z]"),
+			expected: []ReLiteral{{Value: []byte("easy-prefix-"), Foldable: false}},
+			fn:       prefix,
+		},
+		{
+			re:       parse("(video|vodka)-prefix-[a-zA-Z]"),
+			expected: []ReLiteral{{Value: []byte("v"), Foldable: false}},
+			fn:       prefix,
+		},
+		{
+			re:       parse("(?i)prefix-[a-zA-Z]"),
+			expected: []ReLiteral{{Value: []byte("PREFIX-"), Foldable: true}},
+			fn:       prefix,
+		},
+		{
+			re:       parse("(no|prefix)-suffix"),
+			expected: []ReLiteral{},
+			fn:       prefix,
+		},
+
+		{
+			re:       parse("simple"),
+			expected: []ReLiteral{{Value: []byte("simple"), Foldable: false}},
+			fn:       suffix,
+		},
+		{
+			re:       parse("((a)b)"),
+			expected: []ReLiteral{{Value: []byte("ab"), Foldable: false}},
+			fn:       suffix,
+		},
+		{
+			re:       parse("prefix-[a-zA-Z]-suffix"),
+			expected: []ReLiteral{{Value: []byte("-suffix"), Foldable: false}},
+			fn:       suffix,
+		},
+		{
+			re:       parse("suffix-(no|literal)"),
+			expected: []ReLiteral{},
+			fn:       suffix,
+		},
+
+		{
+			re: parse("(something|nothing)-in-(a|the)-way-(song)?"),
+			expected: []ReLiteral{
+				{Value: []byte("-in-"), Foldable: false},
+				{Value: []byte("-way-"), Foldable: false},
+			},
+			fn: middle,
+		},
+		{
+			re: parse("([a-z]+-one-([0-9]-two))-three"),
+			expected: []ReLiteral{
+				{Value: []byte("-one-"), Foldable: false},
+				{Value: []byte("-two-three"), Foldable: false},
+			},
+			fn: middle,
+		},
+	}
+
+	for _, cc := range cases {
+		assert.Equal(t, cc.expected, cc.fn(cc.re))
+	}
+}