From 97f7ca924f550afdc0e8e5153138f0c3cb312e45 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Aug 2026 20:09:03 +0100 Subject: [PATCH 1/2] Use dependents library for discovery --- go.mod | 16 +- go.sum | 50 +++++- internal/discover/analyze.go | 242 +++++++-------------------- internal/discover/analyze_test.go | 243 +++++++-------------------- internal/discover/discover.go | 259 +++++++++++++++++------------ internal/discover/discover_test.go | 11 +- internal/discover/ecosystems.go | 1 + 7 files changed, 335 insertions(+), 487 deletions(-) diff --git a/go.mod b/go.mod index 5b37f25..019aa63 100644 --- a/go.mod +++ b/go.mod @@ -5,21 +5,35 @@ go 1.26.4 require ( github.com/BurntSushi/toml v1.6.0 github.com/git-pkgs/brief v0.9.4 + github.com/git-pkgs/dependents v0.0.0-20260810185351-ddfe65bb8fe8 github.com/git-pkgs/managers v0.10.1 github.com/git-pkgs/manifests v0.6.1 github.com/spf13/cobra v1.10.2 ) require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/bazelbuild/buildtools v0.0.0-20260622120422-77b9b380c0a4 // indirect + github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect + github.com/git-pkgs/clone v0.2.1 // indirect + github.com/git-pkgs/enrichment v0.6.5 // indirect github.com/git-pkgs/licensecheck v0.4.1 // indirect + github.com/git-pkgs/magic v0.1.0 // indirect + github.com/git-pkgs/packageurl-go v0.3.1 // indirect github.com/git-pkgs/pom v0.1.5 // indirect github.com/git-pkgs/purl v0.1.15 // indirect - github.com/git-pkgs/spdx v0.1.4 // indirect + github.com/git-pkgs/registries v0.6.4 // indirect + github.com/git-pkgs/spdx v0.3.0 // indirect github.com/git-pkgs/vers v0.3.0 // indirect + github.com/git-pkgs/vulns v0.2.1 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/oapi-codegen/nullable v1.2.0 // indirect + github.com/oapi-codegen/runtime v1.6.0 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect + github.com/pandatix/go-cvss v0.6.2 // indirect + github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/spf13/pflag v1.0.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 87fed2a..4f0e233 100644 --- a/go.sum +++ b/go.sum @@ -1,44 +1,84 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/bazelbuild/buildtools v0.0.0-20260622120422-77b9b380c0a4 h1:zeHhy1A/3rpHyenrSFE6TgQixKDzw9ZNM1dEHwAdN4I= github.com/bazelbuild/buildtools v0.0.0-20260622120422-77b9b380c0a4/go.mod h1:PLNUetjLa77TCCziPsz0EI8a6CUxgC+1jgmWv0H25tg= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= +github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= github.com/git-pkgs/brief v0.9.4 h1:i6jqzavPAt5QNuA7JzTkenuNB25/CTPp77fRZvqZqVQ= github.com/git-pkgs/brief v0.9.4/go.mod h1:j7qjRMVHVAniVpZff4/Dbg79kIUDJRCuXZJ9H2xrgF4= +github.com/git-pkgs/clone v0.2.1 h1:9Hl3UgMpGwYGlsUYR2KbMexISMTCDV5/1L7r1FFA/lw= +github.com/git-pkgs/clone v0.2.1/go.mod h1:lgbobKgJ6XbPZPsbn4iK0fVskYpFr4stjKKCeG9RsRc= +github.com/git-pkgs/dependents v0.0.0-20260810185351-ddfe65bb8fe8 h1:QgLZ9qVJzBkgr+0sPSrfbjOt9hrJ+/BBpAtmbVULz6E= +github.com/git-pkgs/dependents v0.0.0-20260810185351-ddfe65bb8fe8/go.mod h1:SsxtW3lW2gzH8z5dw/NKNPvASWMfyNoDLdN4roMZQ10= +github.com/git-pkgs/enrichment v0.6.5 h1:U0SPzWVGoK4R8TwojCTASBRTEV+QSs0IitdLmzI/g/k= +github.com/git-pkgs/enrichment v0.6.5/go.mod h1:Vt2PLMvWPOio9DLyC8Gdhh1yxsHwcRcG+L2Kkc9+kak= github.com/git-pkgs/licensecheck v0.4.1 h1:b5ilmpIpgeeewBFjdhJ4W7jvwPIFsYQ7ujZma7sli6k= github.com/git-pkgs/licensecheck v0.4.1/go.mod h1:cfFO7yHHPeuXsoODHBWyevajH2yWcbfkIFELyG3ZpU0= +github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= +github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= github.com/git-pkgs/managers v0.10.1 h1:T877XtuXoJNweDTMGj1H6AOeVolDO44IvsgDN5jq3Ko= github.com/git-pkgs/managers v0.10.1/go.mod h1:8DR7tIQEEyPyJ7QGzStVbovnGl7tAJcrhQLzjQPzFzc= github.com/git-pkgs/manifests v0.6.1 h1:lkdtkipyFmJHJVUktkQH7TzliJbYFigCS7kczJHQxFA= github.com/git-pkgs/manifests v0.6.1/go.mod h1:za7j4NTkJQ/mk8H5m6Na93dMftGsZrHAYaZzGJNUPQs= +github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= +github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4= github.com/git-pkgs/purl v0.1.15/go.mod h1:PqCLVBDeZrZgHysR803/AntMELgIr2LFZVNCcwLH2m0= -github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= -github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= +github.com/git-pkgs/registries v0.6.4 h1:Kq/KlStjaQyE83UXT/tKuzCrIzc4keGeBjtroMqgoHA= +github.com/git-pkgs/registries v0.6.4/go.mod h1:YkGHbxHIe2Ha/ROH6zNkS5PJUUoa9g0Ti/s2XhZnrak= +github.com/git-pkgs/spdx v0.3.0 h1:AN0guJE7vN5gbOMi9We4j1ziS4cwgFVhTvjVDGfaC7Q= +github.com/git-pkgs/spdx v0.3.0/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= +github.com/git-pkgs/vulns v0.2.1 h1:tWGhOfPVDZwkM2Y9vRkMpMR+gjtlu2jhERS5JeNBoKQ= +github.com/git-pkgs/vulns v0.2.1/go.mod h1:/0gHKHQR5SWttZVEMqgOvCXssKFwAtbac/PfkhBax9o= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= +github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= +github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= +github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/package-url/packageurl-go v0.1.6 h1:YO3p6u1XmCUliivUg/qWphaY8vI6hxSnnPv7Bfg3m5M= github.com/package-url/packageurl-go v0.1.6/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +github.com/pandatix/go-cvss v0.6.2 h1:TFiHlzUkT67s6UkelHmK6s1INKVUG7nlKYiWWDTITGI= +github.com/pandatix/go-cvss v0.6.2/go.mod h1:jDXYlQBZrc8nvrMUVVvTG8PhmuShOnKrxP53nOFkt8Q= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/discover/analyze.go b/internal/discover/analyze.go index 12276b6..cc3fd80 100644 --- a/internal/discover/analyze.go +++ b/internal/discover/analyze.go @@ -1,19 +1,9 @@ package discover import ( - "bytes" "context" - "fmt" - "io/fs" - "os" - "os/exec" - "path/filepath" - "sort" - "strings" - "sync" - "github.com/git-pkgs/brief" - "github.com/git-pkgs/brief/kb" + "github.com/git-pkgs/dependents" ) // AnalyzeOptions controls phase-two scoring. @@ -22,201 +12,87 @@ type AnalyzeOptions struct { Workdir string // clones go under here; temp dir if empty Limit int // final number to keep after re-ranking Keep bool // leave clones for a follow-up downstream test + Checkout dependents.Checkout } -// Analyze shallow-clones each candidate, counts test files and source -// files that reference the upstream package, drops candidates with no -// tests or no references, re-ranks, and returns the top Limit. Clone -// failures demote the candidate (Analyzed stays false) rather than -// aborting the run. -func Analyze(ctx context.Context, cands []Candidate, opts AnalyzeOptions, log func(string, ...any)) ([]Candidate, error) { +// Analyze delegates checkout and source analysis to dependents, then applies +// downstream's requirement that analyzed repositories contain tests and +// references to the upstream package. Checkout failures retain their +// popularity score instead of aborting discovery. +func Analyze(ctx context.Context, candidates []Candidate, opts AnalyzeOptions, log func(string, ...any)) ([]Candidate, error) { if log == nil { log = func(string, ...any) {} } if opts.Limit <= 0 { - opts.Limit = len(cands) + opts.Limit = len(candidates) } - workdir := opts.Workdir - if workdir == "" { - dir, err := os.MkdirTemp("", "downstream-analyze-") - if err != nil { - return nil, err - } - workdir = dir - if !opts.Keep { - defer func() { _ = os.RemoveAll(dir) }() - } - } else if err := os.MkdirAll(workdir, analyzeDirPerm); err != nil { - return nil, err - } - - out := make([]Candidate, 0, len(cands)) - for _, c := range cands { - dest := filepath.Join(workdir, slug(c)) - if err := shallowClone(ctx, c.Repo, dest); err != nil { - log("analyze %s: clone failed (%v); keeping with phase-one score", c.Name, err) - out = append(out, c) - continue - } - c.TestFiles, c.ImportFiles = scan(dest, opts.Upstream) - c.Analyzed = true - log("analyze %s: %d test files, %d reference upstream", c.Name, c.TestFiles, c.ImportFiles) - if c.TestFiles == 0 { - log("drop %s: no tests", c.Name) - continue - } - if c.ImportFiles == 0 { - log("drop %s: no files reference %s (stale listing or wrong subpackage?)", c.Name, opts.Upstream) - continue - } - out = append(out, c) + shared := make([]dependents.Candidate, len(candidates)) + for i, candidate := range candidates { + shared[i] = candidate.shared() } - - sort.Slice(out, func(i, j int) bool { return out[i].Score() > out[j].Score() }) - if len(out) > opts.Limit { - out = out[:opts.Limit] - } - return out, nil -} - -const ( - analyzeDirPerm os.FileMode = 0o755 - maxScanSize = 256 << 10 // skip files larger than this for content matching -) - -func shallowClone(ctx context.Context, url, dest string) error { - if fi, err := os.Stat(dest); err == nil && fi.IsDir() { - return nil // reuse a prior clone - } - cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--", url, dest) - out, err := cmd.CombinedOutput() + result, err := dependents.Analyze(ctx, shared, dependents.AnalyzeOptions{ + Upstreams: []string{opts.Upstream}, + Workdir: opts.Workdir, + Checkout: opts.Checkout, + Keep: opts.Keep, + }) if err != nil { - return fmt.Errorf("git clone %s: %w: %s", url, err, strings.TrimSpace(string(out))) - } - return nil -} - -// testDirs returns the set of directory names that conventionally -// hold tests, sourced from brief's knowledge base. -var testDirs = sync.OnceValue(func() map[string]bool { - dirs := map[string]bool{} - if k, err := kb.Load(brief.KnowledgeFS); err == nil { - for _, d := range k.Layouts.Layout.TestDirs { - dirs[d] = true - } + return nil, err } - return dirs -}) -// isTestFile reports whether a file's basename follows a test-naming -// convention. These are test-framework conventions rather than -// per-language rules; the list should move to brief's _layout.toml -// alongside test_dirs. -func isTestFile(base string) bool { - stem, ext, ok := strings.Cut(base, ".") - if !ok { - return false - } - if strings.HasSuffix(stem, "_test") || strings.HasSuffix(stem, "_spec") || - strings.HasPrefix(stem, "test_") { - return true + failures := make(map[string]error, len(result.Failures)) + for _, failure := range result.Failures { + failures[failure.Repository] = failure.Err } - // foo.test.js, foo.spec.ts: the first cut left "test.js" in ext. - return strings.HasPrefix(ext, "test.") || strings.HasPrefix(ext, "spec.") -} -// scan walks dir counting test files and files whose content mentions -// the upstream package name. Both counts are ranking signals so -// precision matters less than consistency across candidates. -func scan(dir, upstream string) (testFiles, importFiles int) { - testDirSet := testDirs() - needle := []byte(upstream) - - _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return nil //nolint:nilerr // best effort - } - name := d.Name() - if d.IsDir() { - if skipDir(name) { - return fs.SkipDir - } - return nil - } - - underTestDir := inTestDir(path, dir, testDirSet) - if underTestDir || isTestFile(name) { - testFiles++ - } - if fileMentions(path, d, needle) { - importFiles++ + filtered := make([]Candidate, 0, len(candidates)) + for i, sharedCandidate := range result.Candidates { + candidate := candidates[i] + updateFromShared(&candidate, sharedCandidate) + if failure := failures[candidate.Repo]; failure != nil { + log("analyze %s: analysis failed (%v); keeping with phase-one score", candidate.Name, failure) + filtered = append(filtered, candidate) + continue } - return nil - }) - return testFiles, importFiles -} -func skipDir(name string) bool { - switch name { - case "vendor", "testdata", "node_modules", "target", "dist", "build": - return true - } - return strings.HasPrefix(name, ".") && name != "." -} - -func inTestDir(path, root string, testDirSet map[string]bool) bool { - rel, err := filepath.Rel(root, filepath.Dir(path)) - if err != nil { - return false - } - for seg := range strings.SplitSeq(rel, string(filepath.Separator)) { - if testDirSet[seg] { - return true + log("analyze %s: %d test files, %d reference upstream", candidate.Name, candidate.TestFiles, candidate.ImportFiles) + _, rejected := dependents.Filter([]dependents.Candidate{sharedCandidate}, dependents.FilterOptions{ + RequireTests: true, + RequireImports: true, + }) + if len(rejected) > 0 { + logAnalysisRejection(log, candidate, rejected[0].Reason, opts.Upstream) + continue } + filtered = append(filtered, candidate) } - return false -} -// nonSourceExt lists extensions whose contents aren't worth scanning -// for references to the upstream package: assets, archives, compiled -// artefacts, lockfiles. This is infrastructure filtering, not -// per-ecosystem knowledge. -var nonSourceExt = map[string]bool{ - ".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".svg": true, - ".ico": true, ".webp": true, ".pdf": true, ".woff": true, ".woff2": true, - ".ttf": true, ".eot": true, ".otf": true, ".mp3": true, ".mp4": true, - ".webm": true, ".ogg": true, ".wav": true, - ".zip": true, ".tar": true, ".gz": true, ".bz2": true, ".xz": true, - ".7z": true, ".rar": true, - ".exe": true, ".dll": true, ".so": true, ".dylib": true, ".a": true, - ".o": true, ".class": true, ".jar": true, ".war": true, ".wasm": true, - ".pyc": true, ".pyo": true, - ".lock": true, ".sum": true, - ".min.js": true, ".min.css": true, + return rankCandidates(filtered, opts.Limit), nil } -func fileMentions(path string, d fs.DirEntry, needle []byte) bool { - ext := strings.ToLower(filepath.Ext(path)) - if nonSourceExt[ext] { - return false - } - // Catch .min.js / .min.css which Ext() reports as .js / .css. - lower := strings.ToLower(d.Name()) - if strings.HasSuffix(lower, ".min.js") || strings.HasSuffix(lower, ".min.css") { - return false +func rankCandidates(candidates []Candidate, limit int) []Candidate { + shared := make([]dependents.Candidate, len(candidates)) + byRepository := make(map[string]Candidate, len(candidates)) + for i, candidate := range candidates { + shared[i] = candidate.shared() + byRepository[candidate.Repo] = candidate } - if info, err := d.Info(); err != nil || info.Size() > maxScanSize { - return false + ranked := dependents.Rank(shared, limit, nil) + out := make([]Candidate, 0, len(ranked)) + for _, candidate := range ranked { + out = append(out, byRepository[candidate.Repository]) } - b, err := os.ReadFile(path) - if err != nil { - return false - } - return bytes.Contains(b, needle) + return out } -func slug(c Candidate) string { - d := c.Dependent() - return d.Slug() +func logAnalysisRejection(log func(string, ...any), candidate Candidate, reason, upstream string) { + switch reason { + case dependents.ReasonNoTests: + log("drop %s: no tests", candidate.Name) + case dependents.ReasonNoImports: + log("drop %s: no files reference %s (stale listing or wrong subpackage?)", candidate.Name, upstream) + default: + log("drop %s: %s", candidate.Name, reason) + } } diff --git a/internal/discover/analyze_test.go b/internal/discover/analyze_test.go index 9e4f214..acd9e75 100644 --- a/internal/discover/analyze_test.go +++ b/internal/discover/analyze_test.go @@ -2,246 +2,121 @@ package discover import ( "context" + "errors" "os" "path/filepath" "strings" "testing" + + "github.com/git-pkgs/dependents" ) func writeTree(t *testing.T, root string, files map[string]string) { t.Helper() for name, body := range files { - p := filepath.Join(root, name) - if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } } } -func TestScanGo(t *testing.T) { - dir := t.TempDir() - writeTree(t, dir, map[string]string{ - "a.go": `package a; import "example.test/upstream"`, - "a_test.go": `package a; import "testing"`, - "sub/b.go": `package sub; import "example.test/upstream/sub"`, - "sub/b_test.go": `package sub; import _ "example.test/upstream"`, - "unrelated/c.go": `package unrelated; import "fmt"`, - "vendor/x/x.go": `package x; import "example.test/upstream"`, - "vendor/x/x_test.go": `package x`, - "testdata/y_test.go": `package y`, - ".git/hooks/z_test.go": `package z`, - "node_modules/w.go": `package w; import "example.test/upstream"`, - "go.sum": "example.test/upstream v1.0.0 h1:xxx\n", - }) - - tests, imports := scan(dir, "example.test/upstream") - if tests != 2 { - t.Errorf("test files = %d, want 2 (a_test.go, sub/b_test.go; vendor/testdata/.git skipped)", tests) - } - // Content-match now: a.go, sub/b.go, sub/b_test.go. go.sum is - // skipped by extension; vendor/node_modules by directory. - if imports != 3 { - t.Errorf("import files = %d, want 3", imports) - } -} - -func TestScanCargo(t *testing.T) { - dir := t.TempDir() - writeTree(t, dir, map[string]string{ - "Cargo.toml": "[package]\nname = \"dep\"\n[dependencies]\nitoa = \"1\"\n", - "src/lib.rs": "use itoa::Buffer;\npub fn f() {}\n", - "src/ser.rs": "let b = itoa::Buffer::new();\n", - "src/unrelated.rs": "use std::fmt;\n", - "tests/integration.rs": "use dep::f;\n#[test] fn t() { f(); }\n", - "tests/api.rs": "use itoa;\n", - "benches/bench.rs": "use itoa::Buffer;\n", - "target/debug/bin": "itoa itoa itoa", - "Cargo.lock": "[[package]]\nname = \"itoa\"\n", - }) - - tests, imports := scan(dir, "itoa") - // tests/ is a brief KB test dir; both files under it count. - if tests != 2 { - t.Errorf("test files = %d, want 2 (tests/integration.rs, tests/api.rs)", tests) - } - // Cargo.toml, src/lib.rs, src/ser.rs, tests/api.rs, benches/bench.rs. - // target/ skipped; Cargo.lock skipped by .lock extension. - if imports != 5 { - t.Errorf("import files = %d, want 5", imports) - } -} - -func TestScanJS(t *testing.T) { - dir := t.TempDir() - writeTree(t, dir, map[string]string{ - "package.json": `{"dependencies":{"lodash":"^4"}}`, - "src/index.js": `import _ from 'lodash'`, - "src/index.test.js": `import { f } from './index'`, - "src/util.spec.ts": `describe('util', () => {})`, - "__tests__/api.js": `const _ = require('lodash')`, - "spec/e2e.js": `it('works')`, - "dist/bundle.min.js": `lodash lodash lodash`, - "node_modules/lodash/x.js": `module.exports = {}`, - "assets/logo.png": "lodash", - }) - - tests, imports := scan(dir, "lodash") - // index.test.js, util.spec.ts, __tests__/api.js, spec/e2e.js - if tests != 4 { - t.Errorf("test files = %d, want 4", tests) - } - // package.json, src/index.js, __tests__/api.js. dist/ and - // node_modules/ skipped; .png and .min.js skipped. - if imports != 3 { - t.Errorf("import files = %d, want 3", imports) - } -} - -func TestIsTestFile(t *testing.T) { - yes := []string{ - "foo_test.go", "foo_test.rs", "foo_test.py", "foo_test.exs", - "foo.test.js", "foo.test.ts", "bar.test.tsx", - "foo_spec.rb", "bar.spec.js", - "test_foo.py", - } - no := []string{ - "foo.go", "test.go", "testing.rb", "contest.js", - "Makefile", "README", "foo_test", - } - for _, f := range yes { - if !isTestFile(f) { - t.Errorf("isTestFile(%q) = false, want true", f) - } - } - for _, f := range no { - if isTestFile(f) { - t.Errorf("isTestFile(%q) = true, want false", f) - } - } -} - -func TestScanSkipsLargeFiles(t *testing.T) { - dir := t.TempDir() - big := strings.Repeat("upstream ", (maxScanSize/9)+10) - writeTree(t, dir, map[string]string{ - "small.rs": "use upstream;", - "huge.rs": big, - }) - _, imports := scan(dir, "upstream") - if imports != 1 { - t.Errorf("import files = %d, want 1 (huge.rs over size limit)", imports) - } -} - func TestAnalyzeRanksAndFilters(t *testing.T) { - work := t.TempDir() - - // "high" has many references and tests; "low" has few; "notest" - // has references but no tests; "noimport" has tests but doesn't - // mention upstream. Repos are pre-seeded as local dirs so - // shallowClone reuses them instead of cloning. - writeTree(t, filepath.Join(work, "high-high"), map[string]string{ - "a.go": `package a; import "example.test/up"`, - "b.go": `package a; import "example.test/up/x"`, - "c.go": `package a; import "example.test/up/y"`, - "a_test.go": `package a`, - "b_test.go": `package a`, - }) - writeTree(t, filepath.Join(work, "low-low"), map[string]string{ - "a.go": `package a; import "example.test/up"`, - "a_test.go": `package a`, + trees := map[string]map[string]string{ + "https://x/high": { + "a.go": `package a; import "example.test/up"`, + "b.go": `package a; import "example.test/up/x"`, + "c.go": `package a; import "example.test/up/y"`, + "a_test.go": `package a`, + "b_test.go": `package a`, + }, + "https://x/low": { + "a.go": `package a; import "example.test/up"`, + "a_test.go": `package a`, + }, + "https://x/notest": { + "a.go": `package a; import "example.test/up"`, + }, + "https://x/noimport": { + "a.go": `package a; import "fmt"`, + "a_test.go": `package a`, + }, + } + checkout := dependents.CheckoutFunc(func(_ context.Context, repository, destination string) (string, error) { + writeTree(t, destination, trees[repository]) + return "abc123", nil }) - writeTree(t, filepath.Join(work, "notest-notest"), map[string]string{ - "a.go": `package a; import "example.test/up"`, - }) - writeTree(t, filepath.Join(work, "noimport-noimport"), map[string]string{ - "a.go": `package a; import "fmt"`, - "a_test.go": `package a`, - }) - - cands := []Candidate{ + candidates := []Candidate{ {Name: "low/low", Repo: "https://x/low", DependentRepos: 100000}, {Name: "high/high", Repo: "https://x/high", DependentRepos: 1}, {Name: "notest/notest", Repo: "https://x/notest", DependentRepos: 50}, {Name: "noimport/noimport", Repo: "https://x/noimport", DependentRepos: 50}, } - got, err := Analyze(context.Background(), cands, AnalyzeOptions{ + got, err := Analyze(context.Background(), candidates, AnalyzeOptions{ Upstream: "example.test/up", - Workdir: work, + Workdir: t.TempDir(), Limit: 3, + Checkout: checkout, }, nil) if err != nil { t.Fatalf("Analyze: %v", err) } - if len(got) != 2 { - t.Fatalf("got %d, want 2 (notest and noimport dropped)", len(got)) - } - if got[0].Name != "high/high" { - t.Errorf("rank[0] = %s, want high/high (3 referencing files beats popularity)", got[0].Name) + t.Fatalf("got %d, want 2", len(got)) } - if got[0].ImportFiles != 3 || got[0].TestFiles != 2 || !got[0].Analyzed { - t.Errorf("high = %+v", got[0]) + if got[0].Name != "high/high" || got[0].ImportFiles != 3 || got[0].TestFiles != 2 || !got[0].Analyzed { + t.Errorf("rank[0] = %+v", got[0]) } if got[1].Name != "low/low" { t.Errorf("rank[1] = %s, want low/low", got[1].Name) } } -func TestAnalyzeCloneFailureKeepsCandidate(t *testing.T) { - cands := []Candidate{ - {Name: "x/x", Repo: "https://invalid.test/does/not/exist", DependentRepos: 10}, +func TestAnalyzeFailureKeepsCandidate(t *testing.T) { + wantErr := errors.New("checkout failed") + checkout := dependents.CheckoutFunc(func(context.Context, string, string) (string, error) { + return "", wantErr + }) + var logs []string + log := func(format string, args ...any) { + logs = append(logs, format) } - got, err := Analyze(context.Background(), cands, AnalyzeOptions{ - Upstream: "example.test/up", - Workdir: t.TempDir(), - }, nil) + + got, err := Analyze(context.Background(), []Candidate{{ + Name: "x/x", Repo: "https://invalid.test/x", DependentRepos: 10, + }}, AnalyzeOptions{Workdir: t.TempDir(), Checkout: checkout}, log) if err != nil { t.Fatalf("Analyze: %v", err) } if len(got) != 1 || got[0].Analyzed { - t.Fatalf("clone failure should keep candidate with Analyzed=false: %+v", got) + t.Fatalf("analysis failure should keep candidate: %+v", got) + } + if len(logs) != 1 || !strings.Contains(logs[0], "analysis failed") { + t.Fatalf("logs = %v", logs) } } func TestCommentIncludesAnalyzeFields(t *testing.T) { - c := Candidate{ + candidate := Candidate{ Name: "x", Repo: "https://x", Analyzed: true, ImportFiles: 12, TestFiles: 34, DependentRepos: 100, Stars: 5, } - cm := c.Comment() + comment := candidate.Comment() for _, want := range []string{"12 files reference upstream", "34 test files", "100 dependent repos"} { - if !strings.Contains(cm, want) { - t.Errorf("comment missing %q: %s", want, cm) + if !strings.Contains(comment, want) { + t.Errorf("comment missing %q: %s", want, comment) } } } func TestCommentNewMarker(t *testing.T) { - c := Candidate{Name: "x", Repo: "https://x", DependentRepos: 1, New: true} - if !strings.HasPrefix(c.Comment(), "discover (new): ") { - t.Errorf("comment = %q, want (new) prefix", c.Comment()) - } -} - -func TestScoreImportSurfaceBeatsPopularity(t *testing.T) { - popular := Candidate{Analyzed: true, ImportFiles: 1, TestFiles: 1, DependentRepos: 1_000_000, Stars: 100_000} - exercised := Candidate{Analyzed: true, ImportFiles: 5, TestFiles: 1, DependentRepos: 10} - if exercised.Score() <= popular.Score() { - t.Errorf("import surface should outrank popularity once analyzed: %d vs %d", - exercised.Score(), popular.Score()) - } -} - -func TestScoreTestFilesBreaksImportTie(t *testing.T) { - few := Candidate{Analyzed: true, ImportFiles: 3, TestFiles: 2, DependentRepos: 1_000_000} - many := Candidate{Analyzed: true, ImportFiles: 3, TestFiles: 40, DependentRepos: 1} - if many.Score() <= few.Score() { - t.Errorf("test file count should break an import tie: %d vs %d", many.Score(), few.Score()) + candidate := Candidate{Name: "x", Repo: "https://x", DependentRepos: 1, New: true} + if !strings.HasPrefix(candidate.Comment(), "discover (new): ") { + t.Errorf("comment = %q, want (new) prefix", candidate.Comment()) } } diff --git a/internal/discover/discover.go b/internal/discover/discover.go index c14576a..76cc85c 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -1,17 +1,15 @@ -// Package discover finds and ranks dependents of a package via the -// ecosyste.ms API. Phase one is API-only: fetch popularity-sorted -// dependents, drop forks/archived/stale repos using the inline -// repo_metadata, dedupe by repository, and rank. +// Package discover adapts downstream's package-specific ecosyste.ms lookup +// and configuration format to github.com/git-pkgs/dependents. package discover import ( "context" "fmt" "io" - "sort" "strings" "time" + "github.com/git-pkgs/dependents" "github.com/git-pkgs/downstream/internal/config" ) @@ -31,8 +29,9 @@ const ( defaultLimit = 5 ) -// Candidate is a dependent that survived phase-one filtering, with -// phase-two fields filled by Analyze. +// Candidate contains the downstream-specific presentation fields for a +// repository candidate. Selection, ranking, and analysis are delegated to the +// dependents package. type Candidate struct { Name string Repo string @@ -43,55 +42,55 @@ type Candidate struct { PushedAt time.Time Language string - // Phase two (Analyze) - TestFiles int // files under a test dir or matching a test-name pattern - ImportFiles int // source files whose content mentions the upstream package name - Analyzed bool // distinguishes "not analyzed" from "analyzed, zero" - New bool // appended by reconcile, not in the existing file - - // Not yet implemented; placeholder so Comment() shape is stable. - TransitiveReach int // modules in go.sum that also depend on upstream - CIGreen bool - - // dropReason is set on candidates filtered out so the progress - // log can explain why. - dropReason string + TestFiles int + ImportFiles int + Analyzed bool + New bool } -// Score ranks candidates. After Analyze, files that reference the -// upstream and test-file count dominate (a candidate that exercises -// and tests the upstream beats a popular one that barely touches it); -// before Analyze, falls back to popularity. -func (c Candidate) Score() int64 { - base := c.Downloads - if base == 0 { - base = int64(c.DependentRepos)*scoreRepoWeight + int64(c.Stars) - } - if !c.Analyzed { - return base - } - return int64(c.ImportFiles)*scoreImportWeight + - int64(c.TestFiles)*scoreTestWeight + - int64(c.TransitiveReach)*scoreReachWeight + - base/scorePopDamp +func (c Candidate) shared() dependents.Candidate { + return dependents.Candidate{ + Repository: c.Repo, + Packages: []dependents.Package{{ + Name: c.Name, + Downloads: c.Downloads, + DependentRepos: c.DependentRepos, + }}, + RepositoryMetadata: dependents.RepositoryMetadata{ + PushedAt: c.PushedAt, + StargazersCount: c.Stars, + Language: c.Language, + }, + Downloads: c.Downloads, + DependentRepos: c.DependentRepos, + Analysis: dependents.Analysis{ + TestFiles: c.TestFiles, + ImportFiles: c.ImportFiles, + }, + Analyzed: c.Analyzed, + } } -const ( - scoreRepoWeight = 10 - scoreImportWeight = 1_000_000 - scoreTestWeight = 200_000 - scoreReachWeight = 100_000 - scorePopDamp = 100 -) +func updateFromShared(candidate *Candidate, shared dependents.Candidate) { + candidate.Repo = shared.Repository + candidate.Stars = shared.RepositoryMetadata.StargazersCount + candidate.DependentRepos = shared.DependentRepos + candidate.Downloads = shared.Downloads + candidate.PushedAt = shared.RepositoryMetadata.PushedAt + candidate.Language = shared.RepositoryMetadata.Language + candidate.TestFiles = shared.Analysis.TestFiles + candidate.ImportFiles = shared.Analysis.ImportFiles + candidate.Analyzed = shared.Analyzed + if candidate.Name == "" && len(shared.Packages) > 0 { + candidate.Name = shared.Packages[0].Name + } +} func (c Candidate) Comment() string { parts := []string{} if c.Analyzed { parts = append(parts, fmt.Sprintf("%d files reference upstream", c.ImportFiles)) parts = append(parts, fmt.Sprintf("%d test files", c.TestFiles)) - if c.TransitiveReach > 0 { - parts = append(parts, fmt.Sprintf("%d transitive consumers", c.TransitiveReach)) - } } if c.DependentRepos > 0 { parts = append(parts, fmt.Sprintf("%d dependent repos", c.DependentRepos)) @@ -127,7 +126,9 @@ func (c Candidate) Dependent() config.Dependent { } } -// Discover runs phase one and returns up to opts.Limit candidates. +// Discover fetches package dependents, then uses the dependents package to +// combine repositories, apply downstream's repository policy, and rank the +// result. func Discover(ctx context.Context, opts Options) ([]Candidate, error) { if opts.Limit <= 0 { opts.Limit = defaultLimit @@ -146,87 +147,127 @@ func Discover(ctx context.Context, opts Options) ([]Candidate, error) { } logf(opts, "querying ecosyste.ms for dependents of %s (%s), pool=%d", opts.Package, opts.Ecosystem, opts.Pool) - pkgs, err := opts.Client.DependentPackages(ctx, opts.Ecosystem, opts.Package, opts.Pool) + packages, err := opts.Client.DependentPackages(ctx, opts.Ecosystem, opts.Package, opts.Pool) if err != nil { return nil, err } - logf(opts, "fetched %d candidates", len(pkgs)) - if len(pkgs) == 0 { + logf(opts, "fetched %d candidates", len(packages)) + if len(packages) == 0 { return nil, fmt.Errorf("no dependents found for %s (%s); the package may not be indexed yet", opts.Package, opts.Ecosystem) } - cands := buildCandidates(pkgs, opts) - kept, dropped := partition(cands, opts) - for _, c := range dropped { - logf(opts, "drop %s: %s", c.Name, c.dropReason) + group, details, dropped := buildGroup(packages, opts) + shared := dependents.Build([]dependents.Group{group}) + + if upstream := opts.upstreamRepo(); upstream != "" { + withoutUpstream := make([]dependents.Candidate, 0, len(shared)) + for _, candidate := range shared { + if len(dependents.ExcludeRepositories([]dependents.Candidate{candidate}, upstream)) == 0 { + logf(opts, "drop %s: same repository as upstream", candidateName(candidate)) + dropped++ + continue + } + withoutUpstream = append(withoutUpstream, candidate) + } + shared = withoutUpstream } - sort.Slice(kept, func(i, j int) bool { return kept[i].Score() > kept[j].Score() }) - if len(kept) > opts.Limit { - kept = kept[:opts.Limit] + + shared, rejected := dependents.Filter(shared, dependents.FilterOptions{ + ExcludeForks: true, + ExcludeArchived: true, + ExcludeMirrors: true, + MaxAge: opts.MaxAge, + }) + for _, rejection := range rejected { + logf(opts, "drop %s: %s", candidateName(rejection.Candidate), rejectionMessage(rejection)) } - logf(opts, "kept %d, dropped %d", len(kept), len(dropped)) - return kept, nil -} + dropped += len(rejected) -func buildCandidates(pkgs []Package, opts Options) []Candidate { - seen := make(map[string]bool, len(pkgs)) - out := make([]Candidate, 0, len(pkgs)) - for _, p := range pkgs { - c := candidateFrom(p, opts) - if c.Repo == "" { - c.dropReason = "no repository_url" - } else if seen[c.Repo] { - continue // monorepo: same repo hosts several packages + shared = dependents.Rank(shared, opts.Limit, nil) + candidates := make([]Candidate, 0, len(shared)) + for _, candidate := range shared { + detail := details[candidate.Repository] + converted := Candidate{ + Name: detail.name, + DependentPackages: detail.dependentPackages, } - seen[c.Repo] = true - out = append(out, c) + updateFromShared(&converted, candidate) + candidates = append(candidates, converted) } - return out + + logf(opts, "kept %d, dropped %d", len(candidates), dropped) + return candidates, nil } -func candidateFrom(p Package, opts Options) Candidate { - c := Candidate{ - Name: p.Name, - Repo: firstNonEmpty(p.RepoMetadata.HTMLURL, p.RepositoryURL), - Stars: p.RepoMetadata.StargazersCount, - DependentPackages: p.DependentPackagesCount, - DependentRepos: p.DependentReposCount, - Downloads: p.Downloads, - PushedAt: p.RepoMetadata.PushedAt, - Language: p.RepoMetadata.Language, - } - c.dropReason = dropReason(p, opts) - if c.Repo == opts.upstreamRepo() { - c.dropReason = "same repository as upstream" - } - return c +type candidateDetail struct { + name string + dependentPackages int } -func dropReason(p Package, opts Options) string { - switch { - case p.RepoMetadata.Fork: - return "fork" - case p.RepoMetadata.Archived: - return "archived" - case p.RepoMetadata.SourceName != "": - return "mirror of " + p.RepoMetadata.SourceName - case p.Status == "removed" || p.Status == "deprecated": - return p.Status - case !p.RepoMetadata.PushedAt.IsZero() && time.Since(p.RepoMetadata.PushedAt) > opts.MaxAge: - return fmt.Sprintf("stale (last push %s)", p.RepoMetadata.PushedAt.Format("2006-01-02")) +func buildGroup(packages []Package, opts Options) (dependents.Group, map[string]candidateDetail, int) { + group := dependents.Group{ + Upstream: dependents.PackageRef{Name: opts.Package, Ecosystem: opts.Ecosystem}, } - return "" -} + details := make(map[string]candidateDetail, len(packages)) + dropped := 0 + for _, pkg := range packages { + repository := firstNonEmpty(pkg.RepoMetadata.HTMLURL, pkg.RepositoryURL) + if repository == "" { + logf(opts, "drop %s: no repository_url", pkg.Name) + dropped++ + continue + } + if pkg.Status == "removed" || pkg.Status == "deprecated" { + logf(opts, "drop %s: %s", pkg.Name, pkg.Status) + dropped++ + continue + } -func partition(cands []Candidate, _ Options) (kept, dropped []Candidate) { - for _, c := range cands { - if c.dropReason == "" { - kept = append(kept, c) - } else { - dropped = append(dropped, c) + detail := details[repository] + if detail.name == "" { + detail.name = pkg.Name } + detail.dependentPackages = max(detail.dependentPackages, pkg.DependentPackagesCount) + details[repository] = detail + + group.Dependents = append(group.Dependents, dependents.Dependent{ + Package: dependents.Package{ + Name: pkg.Name, + Ecosystem: pkg.Ecosystem, + LatestVersion: pkg.LatestRelease, + Downloads: pkg.Downloads, + DependentRepos: pkg.DependentReposCount, + }, + Repository: repository, + RepositoryMetadata: dependents.RepositoryMetadata{ + Fork: pkg.RepoMetadata.Fork, + Archived: pkg.RepoMetadata.Archived, + MirrorURL: pkg.RepoMetadata.MirrorURL, + SourceName: pkg.RepoMetadata.SourceName, + PushedAt: pkg.RepoMetadata.PushedAt, + StargazersCount: pkg.RepoMetadata.StargazersCount, + Language: pkg.RepoMetadata.Language, + }, + }) + } + return group, details, dropped +} + +func candidateName(candidate dependents.Candidate) string { + if len(candidate.Packages) > 0 && candidate.Packages[0].Name != "" { + return candidate.Packages[0].Name + } + return candidate.Repository +} + +func rejectionMessage(rejection dependents.Rejection) string { + if rejection.Reason == dependents.ReasonStale && !rejection.Candidate.RepositoryMetadata.PushedAt.IsZero() { + return fmt.Sprintf("stale (last push %s)", rejection.Candidate.RepositoryMetadata.PushedAt.Format("2006-01-02")) + } + if rejection.Reason == dependents.ReasonMirror && rejection.Candidate.RepositoryMetadata.SourceName != "" { + return "mirror of " + rejection.Candidate.RepositoryMetadata.SourceName } - return kept, dropped + return rejection.Reason } func (o Options) upstreamRepo() string { diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go index 0dd5192..dcba5a2 100644 --- a/internal/discover/discover_test.go +++ b/internal/discover/discover_test.go @@ -253,10 +253,11 @@ func TestDependentPackagesPaging(t *testing.T) { } } -func TestScorePrefersDownloads(t *testing.T) { - a := Candidate{DependentRepos: 1000, Stars: 50000} - b := Candidate{Downloads: 5_000_000} - if b.Score() <= a.Score() { - t.Errorf("downloads should win: a=%d b=%d", a.Score(), b.Score()) +func TestRankPrefersDownloads(t *testing.T) { + popularByRepositories := Candidate{Name: "repos", Repo: "https://x/repos", DependentRepos: 1000, Stars: 50000} + popularByDownloads := Candidate{Name: "downloads", Repo: "https://x/downloads", Downloads: 5_000_000} + ranked := rankCandidates([]Candidate{popularByRepositories, popularByDownloads}, 0) + if ranked[0].Name != "downloads" { + t.Errorf("rank[0] = %s, want downloads", ranked[0].Name) } } diff --git a/internal/discover/ecosystems.go b/internal/discover/ecosystems.go index 008d124..e1dfd12 100644 --- a/internal/discover/ecosystems.go +++ b/internal/discover/ecosystems.go @@ -84,6 +84,7 @@ type RepoMetadata struct { HTMLURL string `json:"html_url"` Fork bool `json:"fork"` Archived bool `json:"archived"` + MirrorURL string `json:"mirror_url"` StargazersCount int `json:"stargazers_count"` PushedAt time.Time `json:"pushed_at"` Language string `json:"language"` From 6d3e80f6ed89f324e935a1d717f60b2fa43a062d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Aug 2026 20:41:28 +0100 Subject: [PATCH 2/2] Use dependents v0.1.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 019aa63..b461040 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.4 require ( github.com/BurntSushi/toml v1.6.0 github.com/git-pkgs/brief v0.9.4 - github.com/git-pkgs/dependents v0.0.0-20260810185351-ddfe65bb8fe8 + github.com/git-pkgs/dependents v0.1.0 github.com/git-pkgs/managers v0.10.1 github.com/git-pkgs/manifests v0.6.1 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index 4f0e233..bd4aacd 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/git-pkgs/brief v0.9.4 h1:i6jqzavPAt5QNuA7JzTkenuNB25/CTPp77fRZvqZqVQ= github.com/git-pkgs/brief v0.9.4/go.mod h1:j7qjRMVHVAniVpZff4/Dbg79kIUDJRCuXZJ9H2xrgF4= github.com/git-pkgs/clone v0.2.1 h1:9Hl3UgMpGwYGlsUYR2KbMexISMTCDV5/1L7r1FFA/lw= github.com/git-pkgs/clone v0.2.1/go.mod h1:lgbobKgJ6XbPZPsbn4iK0fVskYpFr4stjKKCeG9RsRc= -github.com/git-pkgs/dependents v0.0.0-20260810185351-ddfe65bb8fe8 h1:QgLZ9qVJzBkgr+0sPSrfbjOt9hrJ+/BBpAtmbVULz6E= -github.com/git-pkgs/dependents v0.0.0-20260810185351-ddfe65bb8fe8/go.mod h1:SsxtW3lW2gzH8z5dw/NKNPvASWMfyNoDLdN4roMZQ10= +github.com/git-pkgs/dependents v0.1.0 h1:reRRGSxMSb0hrvgNcEoDPFds0g0hOVqEaD5cZ4+vSEU= +github.com/git-pkgs/dependents v0.1.0/go.mod h1:SsxtW3lW2gzH8z5dw/NKNPvASWMfyNoDLdN4roMZQ10= github.com/git-pkgs/enrichment v0.6.5 h1:U0SPzWVGoK4R8TwojCTASBRTEV+QSs0IitdLmzI/g/k= github.com/git-pkgs/enrichment v0.6.5/go.mod h1:Vt2PLMvWPOio9DLyC8Gdhh1yxsHwcRcG+L2Kkc9+kak= github.com/git-pkgs/licensecheck v0.4.1 h1:b5ilmpIpgeeewBFjdhJ4W7jvwPIFsYQ7ujZma7sli6k=