feat: Go 1.17 compatibility via build tags

.github/workflows/test.yml

@@ -7,7 +7,8 @@ on:
     branches: [master, main]
 
 jobs:
-  test:
+  test-modern:
+    name: "Go 1.24 (modern)"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -48,13 +49,13 @@ jobs:
           echo 'JDBC=jdbc:mysql://root:secret@192.168.1.1:3306/prod' >> /tmp/found-ci-test/app.env
 
           # Test text output
-          ./found -i /tmp/found-ci-test -t templates/extract/credential/ -q
+          ./found -i /tmp/found-ci-test -t templates/proton_rules/credential/ -q
 
           # Test JSON output
-          ./found -i /tmp/found-ci-test -t templates/extract/credential/ -o json -q | head -1 | python3 -c "import sys,json; json.load(sys.stdin)"
+          ./found -i /tmp/found-ci-test -t templates/proton_rules/credential/ -o json -q | head -1 | python3 -c "import sys,json; json.load(sys.stdin)"
 
           # Test zombie output
-          ./found -i /tmp/found-ci-test -t templates/extract/credential/ -o zombie -s /tmp/targets.json -q
+          ./found -i /tmp/found-ci-test -t templates/proton_rules/credential/ -o zombie -s /tmp/targets.json -q
           python3 -c "
           import json, sys
           targets = [json.loads(l) for l in open('/tmp/targets.json') if l.strip()]
@@ -65,17 +66,52 @@ jobs:
           "
 
           # Test baseline create + filter
-          ./found -i /tmp/found-ci-test -t templates/extract/credential/ -f /tmp/baseline.json -q
+          ./found -i /tmp/found-ci-test -t templates/proton_rules/credential/ -f /tmp/baseline.json -q
           test -f /tmp/baseline.json && echo "baseline file created"
           # With baseline loaded, known findings should be suppressed
-          NEW_COUNT=$(./found -i /tmp/found-ci-test -t templates/extract/credential/ --baseline /tmp/baseline.json -o json -q | wc -l)
+          NEW_COUNT=$(./found -i /tmp/found-ci-test -t templates/proton_rules/credential/ --baseline /tmp/baseline.json -o json -q | wc -l)
           echo "new findings after baseline: $NEW_COUNT (should be 0)"
 
           # Test --fail-on (should exit 0 since no critical findings with these templates)
-          ./found -i /tmp/found-ci-test -t templates/extract/credential/ --fail-on critical -q
+          ./found -i /tmp/found-ci-test -t templates/proton_rules/credential/ --fail-on critical -q
 
           # Test validate
           ./found --validate -t templates/found/keys/adafruit-key.yaml -q
         env:
           GONOSUMCHECK: "github.com/chainreactors/*"
           GONOSUMDB: "github.com/chainreactors/*"
+
+  test-legacy:
+    name: "Go 1.17 (legacy)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.17"
+
+      - name: Downgrade go.mod for Go 1.17
+        run: |
+          sed -i '/^toolchain /d' go.mod
+          sed -i 's/^go .*/go 1.17/' go.mod
+
+      - name: Build proton library
+        run: go build -v ./proton/file/
+        env:
+          CGO_ENABLED: "0"
+          GONOSUMCHECK: "github.com/chainreactors/*"
+          GONOSUMDB: "github.com/chainreactors/*"
+          GOFLAGS: "-mod=mod"
+
+      - name: Test proton library
+        run: go test -v -count=1 ./proton/file/
+        env:
+          CGO_ENABLED: "0"
+          GONOSUMCHECK: "github.com/chainreactors/*"
+          GONOSUMDB: "github.com/chainreactors/*"
+          GOFLAGS: "-mod=mod"

proton/file/archive_bench_test.go

@@ -1,3 +1,6 @@
+//go:build go1.18
+// +build go1.18
+
 package file
 
 import (

proton/file/bench_spray_vs_proton_test.go

@@ -1,3 +1,6 @@
+//go:build go1.18
+// +build go1.18
+
 package file
 
 import (

proton/file/compat.go

@@ -0,0 +1,175 @@
+//go:build go1.18
+// +build go1.18
+
+package file
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/chainreactors/neutron/operators"
+	"github.com/charlievieth/fastwalk"
+	"github.com/mholt/archives"
+	regexp "github.com/wasilibs/go-re2"
+)
+
+// --- directory walk (fastwalk) ---
+
+func parallelWalk(root string, fn func(path string, d fs.DirEntry, err error) error) error {
+	return fastwalk.Walk(nil, root, fn)
+}
+
+// --- regexp cache (wasilibs/go-re2) ---
+
+var (
+	re2Cache   = make(map[string]*regexp.Regexp)
+	re2CacheMu sync.RWMutex
+
+	re2SliceCache   = make(map[*[]string][]*regexp.Regexp)
+	re2SliceCacheMu sync.RWMutex
+)
+
+func getOrCompileRE2(pattern string) compiledRegexp {
+	re2CacheMu.RLock()
+	if r, ok := re2Cache[pattern]; ok {
+		re2CacheMu.RUnlock()
+		return r
+	}
+	re2CacheMu.RUnlock()
+
+	compiled, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil
+	}
+
+	re2CacheMu.Lock()
+	re2Cache[pattern] = compiled
+	re2CacheMu.Unlock()
+	return compiled
+}
+
+func getOrCompileRE2Slice(patterns *[]string) []*regexp.Regexp {
+	re2SliceCacheMu.RLock()
+	if cached, ok := re2SliceCache[patterns]; ok {
+		re2SliceCacheMu.RUnlock()
+		return cached
+	}
+	re2SliceCacheMu.RUnlock()
+
+	compiled := make([]*regexp.Regexp, 0, len(*patterns))
+	for _, p := range *patterns {
+		r, err := regexp.Compile(p)
+		if err == nil {
+			compiled = append(compiled, r)
+		}
+	}
+
+	re2SliceCacheMu.Lock()
+	re2SliceCache[patterns] = compiled
+	re2SliceCacheMu.Unlock()
+	return compiled
+}
+
+func (request *Request) matchRegexRE2(matcher *operators.Matcher, corpus string) (bool, []string) {
+	regexes := getOrCompileRE2Slice(&matcher.Regex)
+	if len(regexes) == 0 {
+		return false, []string{}
+	}
+
+	isAND := strings.EqualFold(matcher.Condition, "and")
+	var matchedRegexes []string
+
+	for i, re := range regexes {
+		if !re.MatchString(corpus) {
+			if isAND {
+				return matcher.ResultWithMatchedSnippet(false, []string{})
+			}
+			continue
+		}
+
+		currentMatches := re.FindAllString(corpus, -1)
+		if !isAND && !matcher.MatchAll {
+			return matcher.ResultWithMatchedSnippet(true, currentMatches)
+		}
+
+		matchedRegexes = append(matchedRegexes, currentMatches...)
+
+		if len(regexes)-1 == i && !matcher.MatchAll {
+			return matcher.ResultWithMatchedSnippet(true, matchedRegexes)
+		}
+	}
+	if len(matchedRegexes) > 0 && matcher.MatchAll {
+		return matcher.ResultWithMatchedSnippet(true, matchedRegexes)
+	}
+	return matcher.ResultWithMatchedSnippet(false, []string{})
+}
+
+func (request *Request) extractRegexRE2(extractor *operators.Extractor, corpus string) map[string]struct{} {
+	regexes := getOrCompileRE2Slice(&extractor.Regex)
+	results := make(map[string]struct{})
+
+	groupPlusOne := extractor.RegexGroup + 1
+	for _, re := range regexes {
+		matches := re.FindAllStringSubmatch(corpus, -1)
+		for _, match := range matches {
+			if len(match) < groupPlusOne {
+				continue
+			}
+			results[match[extractor.RegexGroup]] = struct{}{}
+		}
+	}
+	return results
+}
+
+// --- archive fallback (mholt/archives) ---
+
+func (s *Scanner) scanArchiveFallback(archivePath string, group *scanGroup) []Finding {
+	f, err := os.Open(archivePath)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	format, _, err := archives.Identify(context.Background(), archivePath, f)
+	if err != nil || format == nil {
+		return nil
+	}
+	f.Seek(0, 0)
+	ex, ok := format.(archives.Extractor)
+	if !ok {
+		return nil
+	}
+	var findings []Finding
+	entries := 0
+	ex.Extract(context.Background(), f, func(ctx context.Context, fi archives.FileInfo) error {
+		if fi.IsDir() || fi.Size() == 0 || fi.Size() > maxArchiveEntrySize {
+			return nil
+		}
+		entryExt := filepath.Ext(fi.Name())
+		if _, deny := alwaysDenyExts[entryExt]; deny {
+			return nil
+		}
+		entries++
+		if entries > maxArchiveEntries {
+			return fmt.Errorf("too many entries")
+		}
+		rc, err := fi.Open()
+		if err != nil {
+			return nil
+		}
+		defer rc.Close()
+		data, err := io.ReadAll(io.LimitReader(rc, maxArchiveEntrySize))
+		if err != nil || len(data) == 0 {
+			return nil
+		}
+		entryPath := fmt.Sprintf("%s:%s", archivePath, fi.Name())
+		findings = append(findings, s.scanData(data, entryPath, group)...)
+		return nil
+	})
+	return findings
+}

proton/file/re2_cache.go → proton/file/compat_1.17.go

@@ -1,13 +1,31 @@
+//go:build !go1.18
+// +build !go1.18
+
 package file
 
 import (
+	"fmt"
+	"io"
+	"io/fs"
+	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 
 	"github.com/chainreactors/neutron/operators"
-	regexp "github.com/wasilibs/go-re2"
+	"github.com/mholt/archiver"
 )
 
+// --- directory walk (filepath.WalkDir) ---
+
+func parallelWalk(root string, fn func(path string, d fs.DirEntry, err error) error) error {
+	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		return fn(path, d, err)
+	})
+}
+
+// --- regexp cache (stdlib regexp) ---
+
 var (
 	re2Cache   = make(map[string]*regexp.Regexp)
 	re2CacheMu sync.RWMutex
@@ -16,7 +34,7 @@ var (
 	re2SliceCacheMu sync.RWMutex
 )
 
-func getOrCompileRE2(pattern string) *regexp.Regexp {
+func getOrCompileRE2(pattern string) compiledRegexp {
 	re2CacheMu.RLock()
 	if r, ok := re2Cache[pattern]; ok {
 		re2CacheMu.RUnlock()
@@ -35,8 +53,6 @@ func getOrCompileRE2(pattern string) *regexp.Regexp {
 	return compiled
 }
 
-// getOrCompileRE2Slice returns a cached compiled regex slice for the given
-// pattern list pointer. This avoids re-creating the slice on every call.
 func getOrCompileRE2Slice(patterns *[]string) []*regexp.Regexp {
 	re2SliceCacheMu.RLock()
 	if cached, ok := re2SliceCache[patterns]; ok {
@@ -47,7 +63,8 @@ func getOrCompileRE2Slice(patterns *[]string) []*regexp.Regexp {
 
 	compiled := make([]*regexp.Regexp, 0, len(*patterns))
 	for _, p := range *patterns {
-		if r := getOrCompileRE2(p); r != nil {
+		r, err := regexp.Compile(p)
+		if err == nil {
 			compiled = append(compiled, r)
 		}
 	}
@@ -108,3 +125,40 @@ func (request *Request) extractRegexRE2(extractor *operators.Extractor, corpus s
 	}
 	return results
 }
+
+// --- archive fallback (mholt/archiver v3) ---
+
+func (s *Scanner) scanArchiveFallback(archivePath string, group *scanGroup) []Finding {
+	ar, _ := archiver.ByExtension(archivePath)
+	if ar == nil {
+		return nil
+	}
+	walker, ok := ar.(archiver.Walker)
+	if !ok {
+		return nil
+	}
+	var findings []Finding
+	entries := 0
+	_ = walker.Walk(archivePath, func(f archiver.File) error {
+		if f.IsDir() || f.Size() == 0 || f.Size() > maxArchiveEntrySize {
+			return nil
+		}
+		entryExt := filepath.Ext(f.Name())
+		if _, deny := alwaysDenyExts[entryExt]; deny {
+			return nil
+		}
+		entries++
+		if entries > maxArchiveEntries {
+			return fmt.Errorf("too many entries")
+		}
+		defer f.Close()
+		data, err := io.ReadAll(io.LimitReader(f, maxArchiveEntrySize))
+		if err != nil || len(data) == 0 {
+			return nil
+		}
+		entryPath := fmt.Sprintf("%s:%s", archivePath, f.Name())
+		findings = append(findings, s.scanData(data, entryPath, group)...)
+		return nil
+	})
+	return findings
+}